diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a12aa73b85..307247651e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,21 +1,141 @@ version: 2.1 orbs: codecov: codecov/codecov@4.0.1 - node: circleci/node@5.1.0 # Add this line to declare the node orb + node: circleci/node@5.1.0 # Add this line to declare the node orb + win: circleci/windows@5.0 # Add Windows orb +commands: + setup_google_dns: + steps: + - run: + name: "Configure Google DNS" + command: | + # Backup original resolv.conf + sudo cp /etc/resolv.conf /etc/resolv.conf.backup + # Add both local and Google DNS servers + echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf + echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf + echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf + setup_litellm_enterprise_pip: + steps: + - run: + name: "Install local version of litellm-enterprise" + command: | + pip install --force-reinstall --no-deps -e enterprise/ + setup_litellm_test_deps: + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + # Use uv for the heavy requirements.txt (10-100x faster than pip) + uv pip install --system -r requirements.txt + # Use pip for test deps (small set, avoids uv strict-resolution + # conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5) + pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \ + "pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \ + "pydantic==2.12.5" "mcp==1.26.0" "requests-mock>=1.12.1" \ + "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \ + "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \ + "a2a" "parameterized>=0.9.0" + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ~/.local/lib + - ~/.local/bin + - ~/.cache/uv + key: v3-litellm-uv-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} jobs: - local_testing: + # Add Windows testing job + using_litellm_on_windows: + executor: + name: win/default + shell: powershell.exe + working_directory: ~/project + steps: + - checkout + - run: + name: Install Python + command: | + choco install python --version=3.11.0 -y --no-progress --force + refreshenv + python --version + environment: + CHOCOLATEY_CONFIRM_ALL: "true" + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + pip install pytest + pip install . + - run: + name: Run Windows-specific test + command: | + python -m pytest tests/windows_tests/test_litellm_on_windows.py -v + + mypy_linting: docker: - - image: cimg/python:3.11 + - image: cimg/python:3.12 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: medium steps: - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip uninstall fastuuid -y + pip install "mypy==1.18.2" + - run: + name: MyPy Type Checking + command: | + cd litellm + # Use the same approach as GitHub Actions, explicitly exclude fastuuid to avoid segfaults + python -m mypy . + cd .. + no_output_timeout: 10m + semgrep: + docker: + - image: cimg/python:3.12 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: medium + steps: + - checkout + - setup_google_dns + - run: + name: Install Semgrep + command: pip install semgrep + - run: + name: Run Semgrep (custom rules only) + command: semgrep scan --config .semgrep/rules . --error + + local_testing_part1: + docker: + - image: cimg/python:3.12 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + parallelism: 4 + steps: + - checkout + - setup_google_dns - run: name: Show git commit hash command: | @@ -29,50 +149,21 @@ jobs: command: | python -m pip install --upgrade pip python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install mypy - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.34.34" - pip install "aioboto3==12.3.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.45.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.66.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.21.1" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.7.1" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "pytest-xdist==3.6.1" - pip install "websockets==13.1.0" + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ + "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \ + "boto3==1.42.80" langchain lunary==0.2.5 \ + "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ + traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ + "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ + "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ + "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ + python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \ + "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \ + "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps pip uninstall posthog -y + - setup_litellm_enterprise_pip - save_cache: paths: - ./venv @@ -91,30 +182,35 @@ jobs: python -m pip install black python -m black . cd .. - - run: - name: Linting Testing - command: | - cd litellm - python -m pip install types-requests types-setuptools types-redis types-PyYAML - if ! python -m mypy . --ignore-missing-imports; then - echo "mypy detected errors" - exit 1 - fi - cd .. # Run pytest and generate JUnit XML report - run: - name: Run tests + name: Run tests (Part 1 - A-M) command: | - pwd - ls - python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4 - no_output_timeout: 120m + mkdir test-results + + # Discover test files (A-M) + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="xargs python -m pytest \ + -vv \ + --cov=litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ + -n 4 \ + --timeout=300 \ + --timeout_method=thread" + no_output_timeout: 15m - run: name: Rename the coverage files command: | - mv coverage.xml local_testing_coverage.xml - mv .coverage local_testing_coverage + mv coverage.xml local_testing_part1_coverage.xml + mv .coverage local_testing_part1_coverage # Store test results - store_test_results: @@ -122,8 +218,103 @@ jobs: - persist_to_workspace: root: . paths: - - local_testing_coverage.xml - - local_testing_coverage + - local_testing_part1_coverage.xml + - local_testing_part1_coverage + local_testing_part2: + docker: + - image: cimg/python:3.12 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + parallelism: 4 + steps: + - checkout + - setup_google_dns + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + + - restore_cache: + keys: + - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r .circleci/requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \ + "mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \ + "boto3==1.42.80" langchain lunary==0.2.5 \ + "azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \ + traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \ + "detect_secrets==1.5.0" "respx==0.22.0" fastapi \ + "gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \ + "apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \ + python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \ + "diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \ + "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps + pip uninstall posthog -y + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - run: + name: Run prisma ./docker/entrypoint.sh + command: | + set +e + chmod +x docker/entrypoint.sh + ./docker/entrypoint.sh + set -e + - run: + name: Black Formatting + command: | + cd litellm + python -m pip install black + python -m black . + cd .. + + # Run pytest and generate JUnit XML report + - run: + name: Run tests (Part 2 - N-Z) + command: | + mkdir test-results + + # Discover test files (N-Z) + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="xargs python -m pytest \ + -vv \ + --cov=litellm \ + --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ + -n 4 \ + --timeout=300 \ + --timeout_method=thread" + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml local_testing_part2_coverage.xml + mv .coverage local_testing_part2_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - local_testing_part2_coverage.xml + - local_testing_part2_coverage langfuse_logging_unit_tests: docker: - image: cimg/python:3.11 @@ -131,10 +322,11 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: medium steps: - checkout - + - setup_google_dns - run: name: Show git commit hash command: | @@ -152,44 +344,44 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install "pytest-cov==5.0.0" - pip install mypy + pip install "mypy==1.18.2" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pyarrow - pip install "boto3==1.34.34" - pip install "aioboto3==12.3.0" + pip install "boto3==1.42.80" pip install langchain pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.45.0" + pip install "azure-identity==1.25.3" + pip install "langfuse==2.59.7" pip install "logfire==0.29.0" pip install numpydoc pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.66.1 + pip install opentelemetry-api==1.28.0 + pip install opentelemetry-sdk==1.28.0 + pip install opentelemetry-exporter-otlp==1.28.0 + pip install openai==1.100.1 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.21.1" + pip install "httpx==0.28.1" + pip install "respx==0.22.0" pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" + pip install "gunicorn==23.0.0" + pip install "anyio==4.8.0" pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" + pip install "apscheduler==3.11.2" pip install "PyGithub==1.59.1" pip install argon2-cffi pip install "pytest-mock==3.12.0" pip install python-multipart pip install google-cloud-aiplatform pip install prometheus-client==0.20.0 - pip install "pydantic==2.7.1" + pip install "pydantic==2.12.5" pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "websockets==13.1.0" + pip install "Pillow==12.1.1" + pip install "jsonschema==4.23.0" + pip install "websockets==15.0.1" + - setup_litellm_enterprise_pip - save_cache: paths: - ./venv @@ -208,122 +400,11 @@ jobs: command: | pwd ls - python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml langfuse_coverage.xml - mv .coverage langfuse_coverage - + python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - langfuse_coverage.xml - - langfuse_coverage - caching_unit_tests: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - - steps: - - checkout - - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - - restore_cache: - keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install mypy - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.34.34" - pip install "aioboto3==12.3.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.45.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.66.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.21.1" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.7.1" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" - pip install "websockets==13.1.0" - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "caching or cache" - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml caching_coverage.xml - mv .coverage caching_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - caching_coverage.xml - - caching_coverage auth_ui_unit_tests: docker: - image: cimg/python:3.11 @@ -334,15 +415,16 @@ jobs: steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" + pip install "pytest-xdist==3.6.1" - save_cache: paths: - ./venv @@ -360,24 +442,13 @@ jobs: command: | pwd ls - python -m pytest -vv tests/proxy_admin_ui_tests -x --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - - run: - name: Rename the coverage files - command: | - mv coverage.xml auth_ui_unit_tests_coverage.xml - mv .coverage auth_ui_unit_tests_coverage + python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - auth_ui_unit_tests_coverage.xml - - auth_ui_unit_tests_coverage litellm_router_testing: # Runs all tests with the "router" keyword docker: - image: cimg/python:3.11 @@ -385,256 +456,131 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - + resource_class: large + parallelism: 4 steps: - checkout - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "respx==0.21.1" - pip install "pytest-cov==5.0.0" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - python -m pytest tests/local_testing tests/router_unit_tests --cov=litellm --cov-report=xml -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_router_coverage.xml - mv .coverage litellm_router_coverage - # Store test results - - store_test_results: - path: test-results - - - persist_to_workspace: - root: . - paths: - - litellm_router_coverage.xml - - litellm_router_coverage - litellm_proxy_security_tests: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - steps: - - checkout - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_security_tests_coverage.xml - mv .coverage litellm_proxy_security_tests_coverage - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_security_tests_coverage.xml - - litellm_proxy_security_tests_coverage - litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - steps: - - checkout - - - run: - name: Show git commit hash - command: | - echo "Git commit hash: $CIRCLE_SHA1" - + - setup_google_dns - restore_cache: keys: - - v1-dependencies-{{ checksum ".circleci/requirements.txt" }} + - v1-router-testing-deps-{{ checksum "requirements.txt" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r .circleci/requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" + pip install "respx==0.22.0" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" - pip install mypy - pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install pyarrow - pip install "boto3==1.34.34" - pip install "aioboto3==12.3.0" - pip install langchain - pip install lunary==0.2.5 - pip install "azure-identity==1.16.1" - pip install "langfuse==2.45.0" - pip install "logfire==0.29.0" - pip install numpydoc - pip install traceloop-sdk==0.21.1 - pip install opentelemetry-api==1.25.0 - pip install opentelemetry-sdk==1.25.0 - pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.66.1 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" - pip install "httpx==0.24.1" - pip install "respx==0.21.1" - pip install fastapi - pip install "gunicorn==21.2.0" - pip install "anyio==4.2.0" - pip install "aiodynamo==23.10.1" - pip install "asyncio==3.4.3" - pip install "apscheduler==3.10.4" - pip install "PyGithub==1.59.1" - pip install argon2-cffi - pip install "pytest-mock==3.12.0" - pip install python-multipart - pip install google-cloud-aiplatform - pip install prometheus-client==0.20.0 - pip install "pydantic==2.7.1" - pip install "diskcache==5.6.1" - pip install "Pillow==10.3.0" - pip install "jsonschema==4.22.0" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps - save_cache: paths: - - ./venv - key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }} - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e + - /home/circleci/.pyenv + - /home/circleci/.local + key: v1-router-testing-deps-{{ checksum "requirements.txt" }} # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls - python -m pytest tests/proxy_unit_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_unit_tests_coverage.xml - mv .coverage litellm_proxy_unit_tests_coverage + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + + echo "$TEST_FILES" | circleci tests run \ + --split-by=timings \ + --verbose \ + --command="xargs python -m pytest \ + -v \ + -k 'router' \ + -n 4 \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + --timeout=300 --timeout_method=thread" + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . + litellm_router_unit_testing: # Runs all tests with the "router" keyword + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: large + + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - v1-router-unit-deps-{{ checksum "requirements.txt" }} + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" + pip install "respx==0.22.0" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + pip install semantic_router --no-deps + pip install aurelio_sdk --no-deps + pip install "pytest-xdist==3.6.1" + - save_cache: paths: - - litellm_proxy_unit_tests_coverage.xml - - litellm_proxy_unit_tests_coverage + - /home/circleci/.pyenv + - /home/circleci/.local + key: v1-router-unit-deps-{{ checksum "requirements.txt" }} + # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4 + no_output_timeout: 15m + # Store test results + - store_test_results: + path: test-results litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - - image: cimg/python:3.11 + - image: cimg/python:3.13.1 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: medium steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + pip install wheel setuptools + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" - pip install "respx==0.21.1" + pip install "respx==0.22.0" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" - pip install "pytest-cov==5.0.0" # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls - python -m pytest tests/local_testing/ -vv -k "assistants" --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_assistants_api_coverage.xml - mv .coverage litellm_assistants_api_coverage - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_assistants_api_coverage.xml - - litellm_assistants_api_coverage - load_testing: - docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - - steps: - - checkout - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - python -m pytest -vv tests/load_tests -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - + python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results @@ -645,32 +591,84 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large steps: - checkout + - setup_google_dns + - restore_cache: + keys: + - v1-llm-translation-deps-{{ checksum "requirements.txt" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" - pip install "respx==0.21.1" + pip install "respx==0.22.0" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + - save_cache: + paths: + - /home/circleci/.pyenv + - /home/circleci/.local + key: v1-llm-translation-deps-{{ checksum "requirements.txt" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + # Add --timeout to kill hanging tests after 120s (2 min) + # Add --durations=20 to show 20 slowest tests for debugging + # Subdirectories with dedicated jobs (maintain this list as new jobs are added) + IGNORE_DIRS=( + "tests/llm_translation/realtime" + ) + IGNORE_ARGS="" + for dir in "${IGNORE_DIRS[@]}"; do + IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" + done + python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + realtime_translation_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets" + # Run pytest and generate JUnit XML report + - run: + name: Run realtime tests + command: | + pwd + ls + # Add --timeout to kill hanging tests after 120s (2 min) + # Add --durations=20 to show 20 slowest tests for debugging + python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread + no_output_timeout: 15m - run: name: Rename the coverage files command: | - mv coverage.xml llm_translation_coverage.xml - mv .coverage llm_translation_coverage + mv coverage.xml realtime_translation_coverage.xml + mv .coverage realtime_translation_coverage # Store test results - store_test_results: @@ -678,8 +676,194 @@ jobs: - persist_to_workspace: root: . paths: - - llm_translation_coverage.xml - - llm_translation_coverage + - realtime_translation_coverage.xml + - realtime_translation_coverage + mcp_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pydantic==2.12.5" + pip install "mcp==1.26.0" + pip install "pytest-xdist==3.6.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml mcp_coverage.xml + mv .coverage mcp_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - mcp_coverage.xml + - mcp_coverage + agent_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pydantic==2.12.5" + pip install "a2a-sdk" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml agent_coverage.xml + mv .coverage agent_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - agent_coverage.xml + - agent_coverage + guardrails_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pydantic==2.12.5" + pip install "boto3==1.42.80" + pip install "semantic_router==0.1.10" --no-deps + pip install aurelio_sdk + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + LITELLM_LOG=WARNING python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml guardrails_coverage.xml + mv .coverage guardrails_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - guardrails_coverage.xml + - guardrails_coverage + + google_generate_content_endpoint_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pydantic==2.12.5" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml google_generate_content_endpoint_coverage.xml + mv .coverage google_generate_content_endpoint_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - google_generate_content_endpoint_coverage.xml + - google_generate_content_endpoint_coverage + llm_responses_api_testing: docker: - image: cimg/python:3.11 @@ -687,42 +871,42 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large steps: - checkout + - setup_google_dns + - restore_cache: + keys: + - v1-llm-responses-deps-{{ checksum "requirements.txt" }} - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" - pip install "respx==0.21.1" + pip install "respx==0.22.0" + pip install "pytest-xdist==3.6.1" + - save_cache: + paths: + - /home/circleci/.pyenv + - /home/circleci/.local + key: v1-llm-responses-deps-{{ checksum "requirements.txt" }} # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/llm_responses_api_testing --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml llm_responses_api_coverage.xml - mv .coverage llm_responses_api_coverage + python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - llm_responses_api_coverage.xml - - llm_responses_api_coverage - litellm_mapped_tests: + ocr_testing: docker: - image: cimg/python:3.11 auth: @@ -732,31 +916,26 @@ jobs: steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest-mock==3.12.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.21.1" - pip install "hypercorn==0.17.3" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + no_output_timeout: 15m - run: name: Rename the coverage files command: | - mv coverage.xml litellm_mapped_tests_coverage.xml - mv .coverage litellm_mapped_tests_coverage + mv coverage.xml ocr_coverage.xml + mv .coverage ocr_coverage # Store test results - store_test_results: @@ -764,8 +943,130 @@ jobs: - persist_to_workspace: root: . paths: - - litellm_mapped_tests_coverage.xml - - litellm_mapped_tests_coverage + - ocr_coverage.xml + - ocr_coverage + search_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml search_coverage.xml + mv .coverage search_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - search_coverage.xml + - search_coverage + # Split litellm_mapped_tests into parallel jobs + litellm_mapped_tests_proxy_part1: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: large + steps: + - setup_litellm_test_deps + - run: + name: Run proxy tests part 1 (high-volume directories) + command: | + prisma generate + export PYTHONUNBUFFERED=1 + python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A + no_output_timeout: 15m + - store_test_results: + path: test-results + litellm_mapped_tests_proxy_part2: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: large + steps: + - setup_litellm_test_deps + - run: + name: Run proxy tests part 2 (all other tests) + command: | + prisma generate + export PYTHONUNBUFFERED=1 + python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A + no_output_timeout: 15m + - store_test_results: + path: test-results + litellm_mapped_enterprise_tests: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: large + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest-mock==3.12.0" + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "hypercorn==0.17.3" + pip install "pydantic==2.12.5" + pip install "mcp==1.26.0" + pip install "requests-mock>=1.12.1" + pip install "responses==0.25.7" + pip install "pytest-xdist==3.6.1" + pip install "semantic_router==0.1.10" --no-deps + pip install aurelio_sdk + pip install "fastapi-offline==1.7.3" + - setup_litellm_enterprise_pip + - run: + name: Run enterprise tests + command: | + pwd + ls + prisma generate + python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4 + no_output_timeout: 15m + # Store test results + - store_test_results: + path: test-results batches_testing: docker: - image: cimg/python:3.11 @@ -776,26 +1077,28 @@ jobs: steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "respx==0.21.1" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "respx==0.22.0" pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install "pytest-cov==5.0.0" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" + pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -820,27 +1123,30 @@ jobs: steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "respx==0.21.1" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install numpydoc + pip install "respx==0.22.0" pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install "pytest-cov==5.0.0" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" - pip install numpydoc + pip install "google-cloud-aiplatform==1.133.0" + pip install pytest-mock + pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -866,24 +1172,21 @@ jobs: steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.21.1" + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-cov==5.0.0" "pytest-asyncio==0.21.1" "respx==0.22.0" "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + no_output_timeout: 15m - run: name: Rename the coverage files command: | @@ -905,41 +1208,33 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: large steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" - pip install "respx==0.21.1" + pip install "respx==0.22.0" + pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml image_gen_coverage.xml - mv .coverage image_gen_coverage - + python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results - - persist_to_workspace: - root: . - paths: - - image_gen_coverage.xml - - image_gen_coverage logging_testing: docker: - image: cimg/python:3.11 @@ -950,33 +1245,39 @@ jobs: steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-cov==5.0.0" pip install "pytest-asyncio==0.21.1" pip install pytest-mock - pip install "respx==0.21.1" + pip install "respx==0.22.0" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install "mlflow==2.17.2" + pip install "anthropic==0.54.0" + pip install "blockbuster==1.5.24" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" # Run pytest and generate JUnit XML report + - setup_litellm_enterprise_pip - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + LITELLM_LOG=WARNING python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread + no_output_timeout: 15m - run: name: Rename the coverage files command: | - mv coverage.xml logging_coverage.xml - mv .coverage logging_coverage + mv coverage.xml logging_coverage.xml || true + mv .coverage logging_coverage || true # Store test results - store_test_results: @@ -986,9 +1287,9 @@ jobs: paths: - logging_coverage.xml - logging_coverage - installing_litellm_on_python: + audio_testing: docker: - - image: circleci/python:3.8 + - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} @@ -996,6 +1297,50 @@ jobs: steps: - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m + - run: + name: Rename the coverage files + command: | + mv coverage.xml audio_coverage.xml + mv .coverage audio_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - audio_coverage.xml + - audio_coverage + installing_litellm_on_python: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns - run: name: Install Dependencies command: | @@ -1006,11 +1351,13 @@ jobs: pip install aiohttp pip install openai pip install click - pip install "boto3==1.34.34" + pip install "boto3==1.42.80" pip install jinja2 - pip install tokenizers=="0.20.0" - pip install uvloop==0.21.0 + pip install "tokenizers==0.22.2" + pip install "uvloop==0.21.0" + pip install "fastuuid==0.14.0" pip install jsonschema + - setup_litellm_enterprise_pip - run: name: Run tests command: | @@ -1025,39 +1372,46 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project + resource_class: medium steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install --upgrade pip uv + pip install wheel setuptools + uv pip install --system -r requirements.txt pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install "pytest-cov==5.0.0" pip install "tomli==2.2.1" + pip install "mcp==1.26.0" - run: name: Run tests command: | pwd ls - python -m pytest -vv tests/local_testing/test_basic_python_version.py + python -m pytest -v tests/local_testing/test_basic_python_version.py helm_chart_testing: machine: - image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker + image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker resource_class: medium working_directory: ~/project steps: - checkout + - attach_workspace: + at: ~/project + - setup_google_dns # Install Helm - run: name: Install Helm command: | curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - + # Install kind - run: name: Install Kind @@ -1065,7 +1419,7 @@ jobs: curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 chmod +x ./kind sudo mv ./kind /usr/local/bin/kind - + # Install kubectl - run: name: Install kubectl @@ -1073,43 +1427,59 @@ jobs: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x kubectl sudo mv kubectl /usr/local/bin/ - + # Create kind cluster - run: name: Create Kind Cluster command: | kind create cluster --name litellm-test - + + - run: + name: Load Docker Database Image for helm tests + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + IMAGE_TAG=${CIRCLE_SHA1:-ci} + docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} + + - run: + name: Load Docker image into Kind + command: | + IMAGE_TAG=${CIRCLE_SHA1:-ci} + kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test + # Run helm lint - run: name: Run helm lint command: | helm lint ./deploy/charts/litellm-helm - + # Run helm tests - run: name: Run helm tests command: | - helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml + IMAGE_TAG=${CIRCLE_SHA1:-ci} + helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \ + --set image.repository=litellm-ci \ + --set image.tag=${IMAGE_TAG} \ + --set image.pullPolicy=Never # Wait for pod to be ready echo "Waiting 30 seconds for pod to be ready..." sleep 30 - + # Print pod logs before running tests echo "Printing pod logs..." kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") - + # Run the helm tests helm test litellm --logs helm test litellm --logs - + # Cleanup - run: name: Cleanup command: | kind delete cluster --name litellm-test - when: always # This ensures cleanup runs even if previous steps fail - + when: always # This ensures cleanup runs even if previous steps fail check_code_and_doc_quality: docker: @@ -1121,6 +1491,7 @@ jobs: steps: - checkout + - setup_google_dns - run: name: Install Dependencies command: | @@ -1135,12 +1506,19 @@ jobs: - run: ruff check ./litellm # - run: python ./tests/documentation_tests/test_general_setting_keys.py - run: python ./tests/code_coverage_tests/check_licenses.py + - run: python ./tests/code_coverage_tests/check_provider_folders_documented.py - run: python ./tests/code_coverage_tests/router_code_coverage.py + - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py + - run: python ./tests/code_coverage_tests/info_log_check.py + - run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py + - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py + - run: python ./tests/code_coverage_tests/code_qa_check_tests.py + - run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py + - run: python ./tests/code_coverage_tests/test_proxy_types_import.py - run: python ./tests/code_coverage_tests/callback_manager_test.py - run: python ./tests/code_coverage_tests/recursive_detector.py - run: python ./tests/code_coverage_tests/test_router_strategy_async.py - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py - - run: python ./tests/code_coverage_tests/bedrock_pricing.py - run: python ./tests/documentation_tests/test_env_keys.py - run: python ./tests/documentation_tests/test_router_settings.py - run: python ./tests/documentation_tests/test_api_docs.py @@ -1148,15 +1526,20 @@ jobs: - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py - run: python ./tests/documentation_tests/test_circular_imports.py - run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py + - run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py + - run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py + - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py + - run: python ./tests/code_coverage_tests/memory_test.py - run: helm lint ./deploy/charts/litellm-helm db_migration_disable_update_check: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: medium working_directory: ~/project steps: - checkout + - setup_google_dns - run: name: Install Python 3.9 command: | @@ -1174,33 +1557,69 @@ jobs: pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install aiohttp + pip install apscheduler - run: - name: Build Docker image + name: Install dockerize command: | - docker build -t myapp . -f ./docker/Dockerfile.database - - run: - name: Run Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ - -e DISABLE_SCHEMA_UPDATE="True" \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - --name my-app \ - myapp:latest \ - --config /app/config.yaml \ - --port 4000 - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Seed database with real schema + command: | + docker run -d \ + -p 4001:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-seed \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Wait for schema seed to complete + command: dockerize -wait http://localhost:4001 -timeout 5m + - run: + name: Stop schema seed container + command: docker stop schema-seed && docker rm schema-seed + - run: + name: Run Docker container with bad schema and disabled updates + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ + -e DISABLE_SCHEMA_UPDATE="True" \ + --name my-app \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ + -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ + -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 - run: name: Wait for container to be ready command: dockerize -wait http://localhost:4000 -timeout 1m @@ -1208,10 +1627,11 @@ jobs: name: Check container logs for expected message command: | echo "=== Printing Full Container Startup Logs ===" - docker logs my-app + LOG_OUTPUT="$(docker logs my-app 2>&1)" + printf '%s\n' "$LOG_OUTPUT" echo "=== End of Full Container Startup Logs ===" - - if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then + + if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then echo "Expected message found in logs. Test passed." else echo "Expected message not found in logs. Test failed." @@ -1220,22 +1640,19 @@ jobs: - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | - python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - no_output_timeout: 120m - + python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: 15m build_and_test: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + - attach_workspace: + at: ~/project + - setup_google_dns - run: name: Install Python 3.9 command: | @@ -1259,12 +1676,11 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" - pip install mypy + pip install "mypy==1.18.2" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pyarrow - pip install "boto3==1.34.34" - pip install "aioboto3==12.3.0" + pip install "boto3==1.42.80" pip install langchain pip install "langfuse>=2.0.0" pip install "logfire==0.29.0" @@ -1272,38 +1688,46 @@ jobs: pip install prisma pip install fastapi pip install jsonschema - pip install "httpx==0.24.1" - pip install "gunicorn==21.2.0" - pip install "anyio==3.7.1" + pip install "httpx==0.28.1" + pip install "gunicorn==23.0.0" + pip install "anyio==4.8.0" pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.66.1" + pip install "openai==1.100.1" + pip install "litellm[proxy]" + pip install "pytest-xdist==3.6.1" - run: - name: Install Grype + name: Install dockerize command: | - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz - run: - name: Build and Scan Docker Images + name: Start PostgreSQL Database command: | - # Build and scan Dockerfile.database - echo "Building and scanning Dockerfile.database..." - docker build -t litellm-database:latest -f ./docker/Dockerfile.database . - grype litellm-database:latest --fail-on high - - # Build and scan main Dockerfile - echo "Building and scanning main Dockerfile..." - docker build -t litellm:latest . - grype litellm:latest --fail-on high + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker tag litellm-docker-database:ci my-app:latest - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e USE_PRISMA_MIGRATE=True \ -e AZURE_API_KEY=$AZURE_API_KEY \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ @@ -1327,6 +1751,7 @@ jobs: -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ my-app:latest \ @@ -1334,13 +1759,10 @@ jobs: --port 4000 \ --detailed_debug \ - run: - name: Install curl and dockerize + name: Install curl command: | sudo apt-get update sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app @@ -1353,8 +1775,8 @@ jobs: command: | pwd ls - python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests - no_output_timeout: 120m + python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + no_output_timeout: 15m # Store test results - store_test_results: @@ -1362,24 +1784,26 @@ jobs: e2e_openai_endpoints: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout + - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: - name: Install Python 3.9 + name: Install Python 3.10 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc - conda create -n myenv python=3.9 -y + conda create -n myenv python=3.10 -y conda activate myenv python --version - run: @@ -1394,39 +1818,62 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" - pip install mypy + pip install "mypy==1.18.2" pip install "jsonlines==4.0.0" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pyarrow - pip install "boto3==1.34.34" - pip install "aioboto3==12.3.0" + pip install "boto3==1.42.80" pip install langchain + pip install "langchain_mcp_adapters==0.0.5" pip install "langfuse>=2.0.0" pip install "logfire==0.29.0" pip install numpydoc pip install prisma pip install fastapi pip install jsonschema - pip install "httpx==0.24.1" - pip install "gunicorn==21.2.0" - pip install "anyio==3.7.1" + pip install "httpx==0.28.1" + pip install "gunicorn==23.0.0" + pip install "anyio==4.8.0" pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.66.1" + pip install "openai==1.100.1" # Run pytest and generate JUnit XML report - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ - -e AZURE_API_KEY=$AZURE_BATCHES_API_KEY \ - -e AZURE_API_BASE=$AZURE_BATCHES_API_BASE \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e AZURE_API_BASE=$AZURE_API_BASE \ -e AZURE_API_VERSION="2024-05-01-preview" \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ @@ -1450,20 +1897,18 @@ jobs: -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \ - my-app:latest \ + litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - run: - name: Install curl and dockerize + name: Install curl command: | sudo apt-get update sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app @@ -1477,7 +1922,7 @@ jobs: pwd ls python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + no_output_timeout: 15m # Store test results - store_test_results: @@ -1485,15 +1930,17 @@ jobs: proxy_logging_guardrails_model_info_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout + - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -1517,12 +1964,11 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" - pip install mypy + pip install "mypy==1.18.2" pip install "google-generativeai==0.3.2" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install pyarrow - pip install "boto3==1.34.34" - pip install "aioboto3==12.3.0" + pip install "boto3==1.42.80" pip install langchain pip install "langfuse>=2.0.0" pip install "logfire==0.29.0" @@ -1530,16 +1976,39 @@ jobs: pip install prisma pip install fastapi pip install jsonschema - pip install "httpx==0.24.1" - pip install "gunicorn==21.2.0" - pip install "anyio==3.7.1" + pip install "httpx==0.28.1" + pip install "gunicorn==23.0.0" + pip install "anyio==4.8.0" pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.66.1" + pip install "openai==1.100.1" - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database - run: name: Run Docker container # intentionally give bad redis credentials here @@ -1547,7 +2016,7 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -1555,22 +2024,20 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \ - my-app:latest \ + litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ @@ -1594,9 +2061,8 @@ jobs: command: | pwd ls - python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: - 120m + python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container @@ -1611,16 +2077,17 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE="bad-license" \ + --add-host host.docker.internal:host-gateway \ --name my-app-3 \ -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ - my-app:latest \ + litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug @@ -1637,25 +2104,138 @@ jobs: - run: name: Run second round of tests command: | - python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - no_output_timeout: 120m + python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: 15m # Store test results - store_test_results: path: test-results + proxy_spend_accuracy_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl + command: | + sudo apt-get update + sudo apt-get install -y curl + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m + # Clean up first container + - run: + name: Stop and remove first container + command: | + docker stop my-app + docker rm my-app proxy_multi_instance_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout + - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -1680,8 +2260,31 @@ jobs: pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database - run: name: Run Docker container 1 # intentionally give bad redis credentials here @@ -1689,7 +2292,7 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -1698,9 +2301,10 @@ jobs: -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ - my-app:latest \ + litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ @@ -1709,7 +2313,7 @@ jobs: command: | docker run -d \ -p 4001:4001 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -1718,9 +2322,10 @@ jobs: -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ + --add-host host.docker.internal:host-gateway \ --name my-app-2 \ -v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \ - my-app:latest \ + litellm-docker-database:ci \ --config /app/config.yaml \ --port 4001 \ --detailed_debug @@ -1748,8 +2353,7 @@ jobs: pwd ls python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: - 120m + no_output_timeout: 15m # Clean up first container # Store test results - store_test_results: @@ -1758,15 +2362,18 @@ jobs: proxy_store_model_in_db_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout + - setup_google_dns - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + sudo systemctl restart docker - run: name: Install Python 3.9 command: | @@ -1792,8 +2399,31 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "assemblyai==0.37.0" - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database - run: name: Run Docker container # intentionally give bad redis credentials here @@ -1801,13 +2431,14 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e STORE_MODEL_IN_DB="True" \ -e LITELLM_MASTER_KEY="sk-1234" \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ - my-app:latest \ + litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ @@ -1832,18 +2463,27 @@ jobs: pwd ls python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: - 120m - # Clean up first container - + no_output_timeout: 15m + - run: + name: Stop and remove containers + command: | + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always + - store_test_results: + path: test-results + proxy_build_from_pip_tests: # Change from docker to machine executor machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout + - setup_google_dns # Remove Docker CLI installation since it's already available in machine executor - run: name: Install Python 3.13 @@ -1859,29 +2499,35 @@ jobs: - run: name: Install Dependencies command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp python -m pip install --upgrade pip - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install mypy + pip install "pytest==7.3.1" "pytest-asyncio==0.21.1" "pytest-retry==1.6.3" \ + "pytest-mock==3.12.0" "mypy==1.18.2" aiohttp apscheduler - run: name: Build Docker image command: | - cd docker/build_from_pip - docker build -t my-app:latest -f Dockerfile.build_from_pip . + docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: | + timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done' - run: name: Run Docker container # intentionally give bad redis credentials here # the OTEL test - should get this as a trace command: | - cd docker/build_from_pip docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -1889,20 +2535,17 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ --name my-app \ - -v $(pwd)/litellm_config.yaml:/app/config.yaml \ + -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ --config /app/config.yaml \ --port 4000 \ @@ -1926,99 +2569,123 @@ jobs: name: Run tests command: | python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - no_output_timeout: - 120m + no_output_timeout: 15m # Clean up first container - run: name: Stop and remove first container command: | - docker stop my-app - docker rm my-app + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: large working_directory: ~/project steps: - checkout + - setup_google_dns - run: - name: Install Docker CLI (In case it's not already installed) - command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io - - run: - name: Install Python 3.9 + name: Install Python 3.10 command: | curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" conda init bash source ~/.bashrc - conda create -n myenv python=3.9 -y + conda create -n myenv python=3.10 -y conda activate myenv python --version - run: name: Install Dependencies command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" - pip install "google-cloud-aiplatform==1.43.0" + pip install "google-cloud-aiplatform==1.133.0" pip install aiohttp - pip install "openai==1.66.1" + pip install "openai==1.100.1" pip install "assemblyai==0.37.0" python -m pip install --upgrade pip - pip install "pydantic==2.7.1" + pip install "pydantic==2.12.5" pip install "pytest==7.3.1" pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" - pip install "boto3==1.34.34" - pip install mypy + pip install "boto3==1.42.80" + pip install "mypy==1.18.2" pip install pyarrow pip install numpydoc pip install prisma pip install fastapi pip install jsonschema pip install "httpx==0.27.0" - pip install "anyio==3.7.1" + pip install "anyio==4.8.0" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" pip install "google-cloud-aiplatform==1.59.0" - pip install "anthropic==0.49.0" - # Run pytest and generate JUnit XML report + pip install "anthropic==0.54.0" + pip install "langchain_mcp_adapters==0.0.5" + pip install "langchain_openai==0.2.1" + pip install "langgraph==0.3.18" + pip install "fastuuid==0.13.5" + pip install -r requirements.txt - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e GEMINI_API_KEY=$GEMINI_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e AZURE_API_BASE=$AZURE_API_BASE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ -v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \ - my-app:latest \ + litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug \ - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs command: docker logs -f my-app @@ -2035,17 +2702,17 @@ jobs: curl -sSL https://rvm.io/mpapis.asc | gpg --import - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - } - + # Install Ruby version manager (RVM) curl -sSL https://get.rvm.io | bash -s stable - + # Source RVM from the correct location source $HOME/.rvm/scripts/rvm - + # Install Ruby 3.2.2 rvm install 3.2.2 rvm use 3.2.2 --default - + # Install latest Bundler gem install bundler @@ -2083,14 +2750,232 @@ jobs: - run: name: Run tests command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv pwd ls - python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m + # Store test results - store_test_results: path: test-results + proxy_e2e_anthropic_messages_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.10 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.10 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install "boto3==1.42.80" + pip install "httpx==0.27.0" + pip install "claude-agent-sdk" + pip install -r requirements.txt + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container with test config + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME="us-east-1" \ + -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run Claude Agent SDK E2E Tests + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + export LITELLM_PROXY_URL="http://localhost:4000" + export LITELLM_API_KEY="sk-1234" + pwd + ls + python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + + proxy_e2e_azure_batches_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.12 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.12 -y + conda activate myenv + python --version + - run: + name: Install Poetry + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + pip install poetry + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=llmproxy \ + -e POSTGRES_PASSWORD=dbpassword9090 \ + -e POSTGRES_DB=litellm \ + -p 5432:5432 \ + postgres:15 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - run: + name: Install system dependencies + command: | + sudo apt-get update -y + sudo apt-get install -y libpq-dev + - run: + name: Install Dependencies + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy" + poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity + - run: + name: Setup litellm-enterprise + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + poetry run pip install --force-reinstall --no-deps -e enterprise/ + - run: + name: Generate Prisma client + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + poetry run prisma generate --schema litellm/proxy/schema.prisma + - run: + name: Run Prisma migrations + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + cd litellm/proxy + poetry run prisma migrate deploy --schema schema.prisma + cd ../.. + - run: + name: Run Azure Batch E2E Tests + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + export USE_LOCAL_LITELLM=true + export USE_MOCK_MODELS=true + export USE_STATE_TRACKER=true + export LITELLM_LOG=DEBUG + poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \ + -vv -s -k "test_e2e_managed_batch" \ + --tb=short \ + --maxfail=3 \ + --durations=10 \ + --junitxml=test-results/junit.xml + no_output_timeout: 15m + upload-coverage: docker: - image: cimg/python:3.9 @@ -2112,172 +2997,256 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine llm_translation_coverage llm_responses_api_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_proxy_security_tests_coverage + coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage coverage xml - codecov/upload: file: ./coverage.xml - publish_to_pypi: + publish_proxy_extras: docker: - image: cimg/python:3.8 - working_directory: ~/project - + working_directory: ~/project/litellm-proxy-extras environment: TWINE_USERNAME: __token__ steps: - - checkout + - checkout: + path: ~/project - run: - name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup + name: Check if litellm-proxy-extras dir or pyproject.toml was modified command: | - cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json + echo "Install TOML package." + python -m pip install toml + # Get current version from pyproject.toml + CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - - run: - name: Check if litellm dir, tests dir, or pyproject.toml was modified - command: | - if [ -n "$(git diff --name-only $CIRCLE_SHA1^..$CIRCLE_SHA1 | grep -E 'pyproject\.toml|litellm/|tests/')" ]; then - echo "litellm, tests, or pyproject.toml updated" + # Get last published version from PyPI + LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])") + + echo "Current version: $CURRENT_VERSION" + echo "Last published version: $LAST_VERSION" + + # Compare versions using Python's packaging.version + VERSION_COMPARE=$(python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)") + + echo "Version compare: $VERSION_COMPARE" + if [ "$VERSION_COMPARE" = "1" ]; then + echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)" + exit 1 + fi + + # If versions are equal or current is greater, check contents + pip download --no-deps litellm-proxy-extras==$LAST_VERSION -d /tmp + + echo "Contents of /tmp directory:" + ls -la /tmp + + # Find the downloaded file (could be .whl or .tar.gz) + DOWNLOADED_FILE=$(ls /tmp/litellm_proxy_extras-*) + echo "Downloaded file: $DOWNLOADED_FILE" + + # Extract based on file extension + if [[ "$DOWNLOADED_FILE" == *.whl ]]; then + echo "Extracting wheel file..." + unzip -q "$DOWNLOADED_FILE" -d /tmp/extracted + EXTRACTED_DIR="/tmp/extracted" else - echo "No changes to litellm, tests, or pyproject.toml. Skipping PyPI publish." + echo "Extracting tar.gz file..." + tar -xzf "$DOWNLOADED_FILE" -C /tmp + EXTRACTED_DIR="/tmp/litellm_proxy_extras-$LAST_VERSION" + fi + + echo "Contents of extracted package:" + ls -R "$EXTRACTED_DIR" + + # Compare contents + if ! diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras; then + if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then + echo "Error: Changes detected in litellm-proxy-extras but version was not bumped" + echo "Current version: $CURRENT_VERSION" + echo "Last published version: $LAST_VERSION" + echo "Changes:" + diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras + exit 1 + fi + else + echo "No changes detected in litellm-proxy-extras. Skipping PyPI publish." circleci step halt fi - run: - name: Checkout code - command: git checkout $CIRCLE_SHA1 - - # Check if setup.py is modified and publish to PyPI - - run: - name: PyPI publish + name: Get new version command: | - echo "Install TOML package." - python -m pip install toml - VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - PACKAGE_NAME=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['name'])") - if ! pip show -v $PACKAGE_NAME | grep -q "Version: ${VERSION}"; then - echo "pyproject.toml modified" - echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc - python -m pip install --upgrade pip - pip install build - pip install wheel - pip install --upgrade twine setuptools - rm -rf build dist + NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") + echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV - echo "Building package" - python -m build - - echo "Twine upload to dist" - echo "Contents of dist directory:" - ls dist/ - twine upload --verbose dist/* - else - echo "Version ${VERSION} of package is already published on PyPI. Skipping PyPI publish." - circleci step halt + - run: + name: Check if versions match + command: | + cd ~/project + # Check pyproject.toml + CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)") + if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then + echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)" + exit 1 fi - - run: - name: Trigger Github Action for new Docker Container + Trigger Load Testing - command: | - echo "Install TOML package." - python3 -m pip install toml - VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - echo "LiteLLM Version ${VERSION}" - curl -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: Bearer $GITHUB_TOKEN" \ - "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ - -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}-nightly\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" - echo "triggering load testing server for version ${VERSION} and commit ${CIRCLE_SHA1}" - curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}&release_type=nightly" - e2e_ui_testing: - machine: - image: ubuntu-2204:2023.10.1 + # Check requirements.txt + REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) + if [ "$REQ_VERSION" != "$NEW_VERSION" ]; then + echo "Error: Version in requirements.txt ($REQ_VERSION) doesn't match new version ($NEW_VERSION)" + exit 1 + fi + + - run: + name: Publish to PyPI + command: | + echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc + python -m pip install --upgrade pip build twine setuptools wheel + rm -rf build dist + python -m build + twine upload --verbose dist/* + + ui_build: + docker: + - image: cimg/node:20.19 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + resource_class: medium+ + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-build-deps-v1- + - restore_cache: + keys: + - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-nextjs-cache-v1- + - run: + name: Install dependencies + command: | + cd ui/litellm-dashboard + npm ci + - save_cache: + key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules + - run: + name: Build UI + command: | + cd ui/litellm-dashboard + source ./build_ui.sh + - save_cache: + key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/.next/cache + - persist_to_workspace: + root: . + paths: + - litellm/proxy/_experimental/out + + ui_unit_tests: + docker: + - image: cimg/node:20.19 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} resource_class: xlarge working_directory: ~/project steps: - checkout + - setup_google_dns + - restore_cache: + keys: + - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + - ui-unit-deps-v1- - run: - name: Build UI + name: Install dependencies command: | - # Set up nvm - export NVM_DIR="/opt/circleci/.nvm" - source "$NVM_DIR/nvm.sh" - source "$NVM_DIR/bash_completion" - - # Install and use Node version - nvm install v18.17.0 - nvm use v18.17.0 - cd ui/litellm-dashboard - - # Install dependencies first - npm install - - # Now source the build script - source ./build_ui.sh + npm ci + - save_cache: + key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} + paths: + - ui/litellm-dashboard/node_modules - run: - name: Install Docker CLI (In case it's not already installed) + name: Run UI unit tests (Vitest) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + cd ui/litellm-dashboard + + CI=true npm run test -- --run \ + --pool forks --poolOptions.forks.maxForks=8 + + build_docker_database_image: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - run: - name: Install Python 3.9 + name: Build Docker image command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version + docker build \ + -t litellm-docker-database:ci \ + -f docker/Dockerfile.database . + + - run: + name: Save Docker image to workspace root + command: | + docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst + + - persist_to_workspace: + root: . + paths: + - litellm-docker-database.tar.zst + + e2e_ui_testing: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: large + working_directory: ~/project + parameters: + browser: + type: string + steps: + - checkout + - setup_google_dns + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database - run: name: Install Dependencies command: | npm install -D @playwright/test - npm install @google-cloud/vertexai - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - pip install "openai==1.66.1" - python -m pip install --upgrade pip - pip install "pydantic==2.7.1" - pip install "pytest==7.3.1" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install mypy - pip install pyarrow - pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema - pip install "httpx==0.24.1" - pip install "anyio==3.7.1" - pip install "asyncio==3.4.3" - run: name: Install Playwright Browsers command: | npx playwright install - - - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . - run: name: Run Docker container command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$SMALL_DATABASE_URL \ + -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \ -e LITELLM_MASTER_KEY="sk-1234" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e UI_USERNAME="admin" \ -e UI_PASSWORD="gm" \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ - --name my-app \ + --name litellm-docker-database-<< parameters.browser >> \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - my-app:latest \ + litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 \ --detailed_debug @@ -2291,7 +3260,7 @@ jobs: sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: name: Start outputting logs - command: docker logs -f my-app + command: docker logs -f litellm-docker-database-<< parameters.browser >> background: true - run: name: Wait for app to be ready @@ -2299,49 +3268,120 @@ jobs: - run: name: Run Playwright Tests command: | - npx playwright test e2e_ui_tests/ --reporter=html --output=test-results - no_output_timeout: 120m - - store_test_results: + npx playwright test \ + --project << parameters.browser >> \ + --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \ + --reporter=html \ + --output=test-results + no_output_timeout: 15m + - store_artifacts: path: test-results + destination: playwright-results - test_nonroot_image: + - store_artifacts: + path: playwright-report + destination: playwright-report + + prisma_schema_sync: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: medium working_directory: ~/project steps: - checkout + - setup_google_dns + - attach_workspace: + at: ~/project - run: - name: Build Docker image + name: Install dockerize command: | - docker build -t non_root_image:latest . -f ./docker/Dockerfile.non_root + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - run: - name: Install Container Structure Test + name: Start PostgreSQL Database command: | - curl -LO https://github.com/GoogleContainerTools/container-structure-test/releases/download/v1.19.3/container-structure-test-linux-amd64 - chmod +x container-structure-test-linux-amd64 - sudo mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_schema_sync \ + -p 5432:5432 \ + postgres:14 - run: - name: Run Container Structure Test + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - run: + name: Load Docker Database Image command: | - container-structure-test test --image non_root_image:latest --config docker/tests/nonroot.yaml + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Run schema sync via prisma db push + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-sync \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Start outputting logs + command: docker logs -f schema-sync + background: true + - run: + name: Wait for proxy to be ready (schema sync complete) + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Stop schema sync container + command: docker stop schema-sync + test_bad_database_url: machine: image: ubuntu-2204:2023.10.1 - resource_class: xlarge + resource_class: medium working_directory: ~/project steps: - checkout + - attach_workspace: + at: ~/project + - setup_google_dns - run: - name: Build Docker image + name: Install dockerize command: | - docker build -t myapp . -f ./docker/Dockerfile.non_root + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker tag litellm-docker-database:ci myapp:latest - run: name: Run Docker container with bad DATABASE_URL command: | docker run --name my-app \ -p 4000:4000 \ + -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ myapp:latest \ --port 4000 > docker_output.log 2>&1 || true @@ -2352,8 +3392,8 @@ jobs: name: Check for expected error command: | if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ - grep -q "httpx.ConnectError: All connection attempts failed" docker_output.log && \ - grep -q "ERROR: Application startup failed. Exiting." docker_output.log; then + (grep -q "Database setup failed after multiple retries" docker_output.log || \ + grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then echo "Expected error found. Test passed." else echo "Expected error not found. Test failed." @@ -2365,7 +3405,31 @@ workflows: version: 2 build_and_test: jobs: - - local_testing: + - using_litellm_on_windows: + filters: + branches: + only: + - main + - /litellm_.*/ + - mypy_linting: + filters: + branches: + only: + - main + - /litellm_.*/ + - semgrep: + filters: + branches: + only: + - main + - /litellm_.*/ + - local_testing_part1: + filters: + branches: + only: + - main + - /litellm_.*/ + - local_testing_part2: filters: branches: only: @@ -2377,22 +3441,14 @@ workflows: only: - main - /litellm_.*/ - - caching_unit_tests: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_unit_testing: - filters: - branches: - only: - main - /litellm_.*/ - - litellm_proxy_security_tests: - filters: - branches: - only: + - main + - /litellm_.*/ + - main + - /litellm_.*/ - main - /litellm_.*/ - litellm_assistants_api_testing: @@ -2407,49 +3463,121 @@ workflows: only: - main - /litellm_.*/ + - litellm_router_unit_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - check_code_and_doc_quality: filters: branches: only: - main - /litellm_.*/ + - ui_build: + filters: + branches: + only: + - main + - /litellm_.*/ + - ui_unit_tests: + requires: + - ui_build + filters: + branches: + only: + - main + - /litellm_.*/ - auth_ui_unit_tests: filters: branches: only: - main - /litellm_.*/ - - e2e_ui_testing: + - build_docker_database_image: filters: branches: only: - main - /litellm_.*/ + - prisma_schema_sync: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + # - e2e_ui_testing: + # name: e2e_ui_testing_chromium + # browser: chromium + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ + # - e2e_ui_testing: + # name: e2e_ui_testing_firefox + # browser: firefox + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ - build_and_test: + requires: + - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - e2e_openai_endpoints: + requires: + - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_logging_guardrails_model_info_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_spend_accuracy_tests: + requires: + - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_multi_instance_tests: + requires: + - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - proxy_store_model_in_db_tests: + requires: + - build_docker_database_image filters: branches: only: @@ -2462,6 +3590,22 @@ workflows: - main - /litellm_.*/ - proxy_pass_through_endpoint_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_e2e_anthropic_messages_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ + - proxy_e2e_azure_batches_tests: filters: branches: only: @@ -2473,18 +3617,82 @@ workflows: only: - main - /litellm_.*/ + - realtime_translation_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - mcp_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - agent_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - guardrails_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - google_generate_content_endpoint_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - llm_responses_api_testing: filters: branches: only: - main - /litellm_.*/ - - litellm_mapped_tests: + - ocr_testing: filters: branches: only: - main - /litellm_.*/ + - search_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_enterprise_tests: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_proxy_part1: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_proxy_part2: + filters: + branches: + only: + - main + - /litellm_.*/ + - main + - /litellm_.*/ + - main + - /litellm_.*/ + - main + - /litellm_.*/ + - main + - /litellm_.*/ + - main + - /litellm_.*/ - batches_testing: filters: branches: @@ -2515,25 +3723,37 @@ workflows: only: - main - /litellm_.*/ + - audio_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - upload-coverage: requires: - - llm_translation_testing - - llm_responses_api_testing - - litellm_mapped_tests + - realtime_translation_testing + - mcp_testing + - agent_testing + - google_generate_content_endpoint_testing + - guardrails_testing + - ocr_testing + - search_testing + - litellm_mapped_tests_proxy_part1 + - litellm_mapped_tests_proxy_part2 + - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing - pass_through_unit_testing - image_gen_testing - logging_testing - - litellm_router_testing - - caching_unit_tests - - litellm_proxy_unit_testing - - litellm_proxy_security_tests + - audio_testing - langfuse_logging_unit_tests - - local_testing + - local_testing_part1 + - local_testing_part2 - litellm_assistants_api_testing - - auth_ui_unit_tests - db_migration_disable_update_check: + requires: + - build_docker_database_image filters: branches: only: @@ -2552,56 +3772,24 @@ workflows: - main - /litellm_.*/ - helm_chart_testing: - filters: - branches: - only: - - main - - /litellm_.*/ - - load_testing: + requires: + - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - test_bad_database_url: + requires: + - build_docker_database_image filters: branches: only: - main - /litellm_.*/ - - publish_to_pypi: - requires: - - local_testing - - build_and_test - - e2e_openai_endpoints - - load_testing - - test_bad_database_url - - llm_translation_testing - - llm_responses_api_testing - - litellm_mapped_tests - - batches_testing - - litellm_utils_testing - - pass_through_unit_testing - - image_gen_testing - - logging_testing - - litellm_router_testing - - caching_unit_tests - - langfuse_logging_unit_tests - - litellm_assistants_api_testing - - auth_ui_unit_tests - - db_migration_disable_update_check - - e2e_ui_testing - - litellm_proxy_unit_testing - - litellm_proxy_security_tests - - installing_litellm_on_python - - installing_litellm_on_python_3_13 - - proxy_logging_guardrails_model_info_tests - - proxy_multi_instance_tests - - proxy_store_model_in_db_tests - - proxy_build_from_pip_tests - - proxy_pass_through_endpoint_tests - - check_code_and_doc_quality + - publish_proxy_extras: filters: branches: only: - main + - /litellm_release_day_.*/ diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index e63fb9dd9a9..be12ab2d0f1 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -1,13 +1,21 @@ # used by CI/CD testing -openai==1.66.1 +openai==1.100.1 python-dotenv tiktoken importlib_metadata cohere -redis +redis==5.2.1 +redisvl==0.4.1 anthropic -orjson==3.9.15 -pydantic==2.7.1 -google-cloud-aiplatform==1.43.0 -fastapi-sso==0.10.0 +orjson==3.10.15 # fast /embedding responses +pydantic==2.12.5 +google-cloud-aiplatform==1.133.0 +google-cloud-iam==2.19.1 +fastapi-sso==0.16.0 uvloop==0.21.0 +mcp==1.26.0 # for MCP server +semantic_router==0.1.10 # for auto-routing with litellm +fastuuid==0.14.0 +responses==0.25.7 # for proxy client tests +pytest-retry==1.6.3 # for automatic test retries +litellm-proxy-extras # for prisma migrations \ No newline at end of file diff --git a/.github/actions/helm-oci-chart-releaser/action.yml b/.github/actions/helm-oci-chart-releaser/action.yml index 059277ed882..454c591d436 100644 --- a/.github/actions/helm-oci-chart-releaser/action.yml +++ b/.github/actions/helm-oci-chart-releaser/action.yml @@ -40,38 +40,55 @@ outputs: runs: using: composite steps: + - name: Helm | Setup + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 + with: + version: v3.20.0 + - name: Helm | Login shell: bash - run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }} env: - HELM_EXPERIMENTAL_OCI: '1' - + REGISTRY_PASSWORD: ${{ inputs.registry_password }} + REGISTRY_USERNAME: ${{ inputs.registry_username }} + REGISTRY: ${{ inputs.registry }} + run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY" + - name: Helm | Dependency if: inputs.update_dependencies == 'true' shell: bash - run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} env: - HELM_EXPERIMENTAL_OCI: '1' + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + run: helm dependency update "$CHART_PATH" - name: Helm | Package shell: bash - run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }} env: - HELM_EXPERIMENTAL_OCI: '1' + CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} + TAG: ${{ inputs.tag }} + APP_VERSION: ${{ inputs.app_version }} + run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION" - name: Helm | Push shell: bash - run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }} env: - HELM_EXPERIMENTAL_OCI: '1' + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}" - name: Helm | Logout shell: bash - run: helm registry logout ${{ inputs.registry }} env: - HELM_EXPERIMENTAL_OCI: '1' + REGISTRY: ${{ inputs.registry }} + run: helm registry logout "$REGISTRY" - name: Helm | Output id: output shell: bash - run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT \ No newline at end of file + env: + REGISTRY: ${{ inputs.registry }} + REPOSITORY: ${{ inputs.repository }} + NAME: ${{ inputs.name }} + TAG: ${{ inputs.tag }} + run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 58e7cfe10da..c49882a8d62 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -4,6 +4,9 @@ updates: directory: "/" schedule: interval: "daily" + cooldown: + default-days: 7 + semver-major-days: 14 groups: github-actions: patterns: diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 00000000000..b4e777969d9 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,35 @@ +# Simple PyPI Publishing + +A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version. + +## How to Use + +1. Go to the **Actions** tab in the GitHub repository +2. Select **Simple PyPI Publish** from the workflow list +3. Click **Run workflow** +4. Enter the version to publish (e.g., `1.74.10`) + +## What the Workflow Does + +1. **Updates** the version in `pyproject.toml` +2. **Copies** the model prices backup file +3. **Builds** the Python package +4. **Publishes** to PyPI + +## Prerequisites + +Make sure the following secret is configured in the repository: +- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication + +## Example Usage + +- Version: `1.74.11` → Publishes as v1.74.11 +- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1 + +## Features + +- ✅ Manual trigger with version input +- ✅ Automatic version updates in `pyproject.toml` +- ✅ Repository safety check (only runs on official repo) +- ✅ Clean package building and publishing +- ✅ Success confirmation with PyPI package link \ No newline at end of file diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml new file mode 100644 index 00000000000..f1ae30e67d7 --- /dev/null +++ b/.github/workflows/_test-unit-base.yml @@ -0,0 +1,96 @@ +name: _Unit Test Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + run: | + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml new file mode 100644 index 00000000000..d53a9e8822a --- /dev/null +++ b/.github/workflows/_test-unit-services-base.yml @@ -0,0 +1,164 @@ +name: _Unit Test Services Base (Reusable) + +on: + workflow_call: + inputs: + test-path: + description: "Pytest path(s) to run" + required: true + type: string + workers: + description: "Number of pytest-xdist workers (0 = no parallelism)" + required: false + type: number + default: 2 + reruns: + description: "Number of reruns for flaky tests" + required: false + type: number + default: 2 + timeout-minutes: + description: "Job timeout in minutes" + required: false + type: number + default: 20 + max-failures: + description: "Stop after this many failures" + required: false + type: number + default: 10 + enable-redis: + description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars" + required: false + type: boolean + default: false + enable-postgres: + description: "Start a local Postgres service container and run Prisma migrations" + required: false + type: boolean + default: false + secrets: + REDIS_HOST: + required: false + REDIS_PORT: + required: false + REDIS_PASSWORD: + required: false + DATABASE_URL: + required: false + POSTGRES_USER: + required: false + POSTGRES_PASSWORD: + required: false + +permissions: + contents: read + +jobs: + run: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + # Environment is derived from the enable-* flags, not caller-controllable. + # This prevents callers from passing arbitrary environment names to bypass secret scoping. + # Note: Postgres service container always starts (GHA limitation), so any Redis job + # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis. + environment: >- + ${{ + inputs.enable-redis && 'integration-redis-postgres' || + inputs.enable-postgres && 'integration-postgres' || + '' + }} + + services: + postgres: + image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 + env: + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + POSTGRES_DB: litellm_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry-services- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run Prisma migrations + if: ${{ inputs.enable-postgres }} + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: | + poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + + - name: Run tests + env: + TEST_PATH: ${{ inputs.test-path }} + MAX_FAILURES: ${{ inputs.max-failures }} + WORKERS: ${{ inputs.workers }} + RERUNS: ${{ inputs.reruns }} + DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} + REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }} + REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }} + REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }} + run: | + if [ "${WORKERS}" = "0" ]; then + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --durations=20 + else + poetry run pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 + fi diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index e7d65242c19..60e89936219 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -2,17 +2,24 @@ name: Updates model_prices_and_context_window.json and Create Pull Request on: schedule: - - cron: "0 0 * * 0" # Run every Sundays at midnight + - cron: "0 0 * * 0" # Run every Sundays at midnight #- cron: "0 0 * * *" # Run daily at midnight +permissions: + contents: write + pull-requests: write + jobs: auto_update_price_and_context_window: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Install Dependencies run: | - pip install aiohttp + pip install 'aiohttp==3.13.3' - name: Update JSON Data run: | python ".github/workflows/auto_update_price_and_context_window_file.py" @@ -25,4 +32,4 @@ jobs: --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ --base main env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} \ No newline at end of file + GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/auto_update_price_and_context_window_file.py b/.github/workflows/auto_update_price_and_context_window_file.py index 3e0731b94bd..461d8d347d9 100644 --- a/.github/workflows/auto_update_price_and_context_window_file.py +++ b/.github/workflows/auto_update_price_and_context_window_file.py @@ -43,8 +43,8 @@ def write_to_file(file_path, data): # Print an error message if writing to file fails print("Error updating JSON file:", e) -# Update the existing models and add the missing models -def transform_remote_data(data): +# Update the existing models and add the missing models for OpenRouter +def transform_openrouter_data(data): transformed = {} for row in data: # Add the fields 'max_tokens' and 'input_cost_per_token' @@ -81,6 +81,34 @@ def transform_remote_data(data): return transformed +# Update the existing models and add the missing models for Vercel AI Gateway +def transform_vercel_ai_gateway_data(data): + transformed = {} + for row in data: + obj = { + "max_tokens": row["context_window"], + "input_cost_per_token": float(row["pricing"]["input"]), + "output_cost_per_token": float(row["pricing"]["output"]), + 'max_output_tokens': row['max_tokens'], + 'max_input_tokens': row["context_window"], + } + + # Handle cache pricing if available + if "pricing" in row: + if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None: + obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}") + + if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None: + obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}") + + mode = "embedding" if "embedding" in row["id"].lower() else "chat" + + obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode}) + + transformed[f'vercel_ai_gateway/{row["id"]}'] = obj + + return transformed + # Load local data from a specified file def load_local_data(file_path): @@ -100,22 +128,32 @@ def load_local_data(file_path): def main(): local_file_path = "model_prices_and_context_window.json" # Path to the local data file - url = "https://openrouter.ai/api/v1/models" # URL to fetch remote data + openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data + vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data # Load local data from file local_data = load_local_data(local_file_path) - # Fetch remote data asynchronously - remote_data = asyncio.run(fetch_data(url)) - # Transform the fetched remote data - remote_data = transform_remote_data(remote_data) + + # Fetch OpenRouter data + openrouter_data = asyncio.run(fetch_data(openrouter_url)) + # Transform the fetched OpenRouter data + openrouter_data = transform_openrouter_data(openrouter_data) + + # Fetch Vercel AI Gateway data + vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url)) + # Transform the fetched Vercel AI Gateway data + vercel_data = transform_vercel_ai_gateway_data(vercel_data) + + # Combine both datasets + all_remote_data = {**openrouter_data, **vercel_data} - # If both local and remote data are available, synchronize and save - if local_data and remote_data: - sync_local_data_with_remote(local_data, remote_data) + # If both local and openrouter data are available, synchronize and save + if local_data and all_remote_data: + sync_local_data_with_remote(local_data, all_remote_data) write_to_file(local_file_path, local_data) else: print("Failed to fetch model data from either local file or URL.") # Entry point of the script if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/.github/workflows/check-schema-sync.yml b/.github/workflows/check-schema-sync.yml new file mode 100644 index 00000000000..0e5e2804e60 --- /dev/null +++ b/.github/workflows/check-schema-sync.yml @@ -0,0 +1,58 @@ +name: Check Schema Sync + +on: + pull_request: + paths: + - 'schema.prisma' + - 'litellm/proxy/schema.prisma' + - 'litellm-proxy-extras/litellm_proxy_extras/schema.prisma' + +permissions: + contents: read + +jobs: + check-sync: + name: Verify schema.prisma copies match root + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout PR + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Check all schemas match root + run: | + EXIT=0 + + diff schema.prisma litellm/proxy/schema.prisma || { + echo "::error file=litellm/proxy/schema.prisma::litellm/proxy/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma || { + echo "::error file=litellm-proxy-extras/litellm_proxy_extras/schema.prisma::litellm-proxy-extras/litellm_proxy_extras/schema.prisma differs from root schema.prisma" + EXIT=1 + } + + if [ "$EXIT" -ne 0 ]; then + echo "" + echo "Schema files are out of sync." + echo "The root schema.prisma is the source of truth." + echo "" + echo "To fix, run from the repo root:" + echo " cp schema.prisma litellm/proxy/schema.prisma" + echo " cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma" + exit 1 + fi + + echo "All schema copies are in sync with root." diff --git a/.github/workflows/check_duplicate_issues.yml b/.github/workflows/check_duplicate_issues.yml new file mode 100644 index 00000000000..289d78880ad --- /dev/null +++ b/.github/workflows/check_duplicate_issues.yml @@ -0,0 +1,53 @@ +name: Check Duplicate Issues + +on: + issues: + types: [opened, edited] + +jobs: + check-duplicate: + runs-on: ubuntu-latest + permissions: + issues: write + contents: read + steps: + - name: Check for potential duplicates + uses: wow-actions/potential-duplicates@4d4ea0352e0383859279938e255179dd1dbb67b5 # v1.1.0 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + label: potential-duplicate + threshold: 0.6 + reaction: eyes + comment: | + **⚠️ Potential duplicate detected** + + This issue appears similar to existing issue(s): + {{#issues}} + - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar) + {{/issues}} + + Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference. + + - name: Checkout close script + if: github.event.action == 'opened' + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + if: github.event.action == 'opened' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Auto-close if high-confidence duplicate + if: github.event.action == 'opened' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 .github/scripts/close_duplicate_issues.py \ + --issue-number ${{ github.event.issue.number }} \ + --repo ${{ github.repository }} \ + --threshold 0.85 \ + --close diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..e86fca17c7a --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,55 @@ +name: "CodeQL" + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + # Run daily at 04:00 UTC + - cron: "0 4 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + analyze: + if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm' + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + security-events: write + packages: read + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: javascript-typescript + build-mode: none + - language: python + build-mode: none + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + config-file: ./.github/codeql/codeql-config.yml + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000000..52d64addea9 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,46 @@ +name: CodSpeed Benchmarks + +on: + push: + branches: + - main + pull_request: + branches: + - main + # Allow CodSpeed to trigger backtest performance analysis + # in order to generate initial data + workflow_dispatch: + +permissions: + contents: read + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + benchmarks: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install -e "." + pip install pytest pytest-codspeed==4.3.0 + + - name: Run benchmarks + uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1 + with: + mode: simulation + run: pytest tests/benchmarks/ --codspeed diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000000..2ae01823a96 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,93 @@ +name: Create Release + +on: + workflow_dispatch: + inputs: + tag: + description: "Release tag (e.g. v1.83.0-stable)" + required: true + type: string + commit_hash: + description: "Full 40-char commit SHA to target" + required: true + type: string + +permissions: {} + +jobs: + release: + name: Create Release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Validate inputs + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + run: | + if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then + echo "::error::commit_hash must be a full 40-character commit SHA" + exit 1 + fi + if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then + echo "::error::tag must start with vX.Y.Z" + exit 1 + fi + + - name: Create release + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const tag = process.env.TAG; + const commitHash = process.env.COMMIT_HASH; + + const cosignSection = [ + `## Verify Docker Image Signature`, + ``, + `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`, + ``, + '```bash', + `cosign verify \\`, + ` --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`, + ` ghcr.io/berriai/litellm:${tag}`, + '```', + ``, + `Expected output:`, + ``, + '```', + `The following checks were performed on each of these signatures:`, + ` - The cosign claims were validated`, + ` - The signatures were verified against the specified public key`, + '```', + ``, + `---`, + ``, + ].join('\n'); + + try { + const response = await github.rest.repos.createRelease({ + draft: true, + generate_release_notes: true, + target_commitish: commitHash, + name: tag, + owner: context.repo.owner, + prerelease: false, + repo: context.repo.repo, + tag_name: tag, + }); + + const updatedBody = cosignSection + (response.data.body ?? ''); + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: response.data.id, + body: updatedBody, + draft: false, + }); + } catch (error) { + core.setFailed(error.message); + } diff --git a/.github/workflows/create_daily_staging_branch.yml b/.github/workflows/create_daily_staging_branch.yml new file mode 100644 index 00000000000..424d8de0a41 --- /dev/null +++ b/.github/workflows/create_daily_staging_branch.yml @@ -0,0 +1,87 @@ +name: Create Daily Staging Branch + +on: + schedule: + - cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC + workflow_dispatch: # Allow manual trigger + +jobs: + create-staging-branch: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Create daily staging branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Configure Git user + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Generate branch name with MM_DD_YYYY format + BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')" + echo "Creating branch: $BRANCH_NAME" + + # Fetch all branches + git fetch --all + + # Check if the branch already exists + if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then + echo "Branch $BRANCH_NAME already exists. Skipping creation." + else + echo "Creating new branch: $BRANCH_NAME" + # Create the new branch from main + git checkout -b $BRANCH_NAME origin/main + # Push the new branch + git push origin $BRANCH_NAME + echo "Successfully created and pushed branch: $BRANCH_NAME" + fi + + create-internal-dev-branch: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Create internal dev branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Configure Git user + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Generate branch name with MM_DD_YYYY format + BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')" + echo "Creating branch: $BRANCH_NAME" + + # Fetch all branches + git fetch --all + + # Check if the branch already exists + if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then + echo "Branch $BRANCH_NAME already exists. Skipping creation." + else + echo "Creating new branch: $BRANCH_NAME" + # Create the new branch from main + git checkout -b $BRANCH_NAME origin/main + # Push the new branch + git push origin $BRANCH_NAME + echo "Successfully created and pushed branch: $BRANCH_NAME" + fi diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml deleted file mode 100644 index 587abc8ea74..00000000000 --- a/.github/workflows/ghcr_deploy.yml +++ /dev/null @@ -1,374 +0,0 @@ -# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM -name: Build, Publish LiteLLM Docker Image. New Release -on: - workflow_dispatch: - inputs: - tag: - description: "The tag version you want to build" - release_type: - description: "The release type you want to build. Can be 'latest', 'stable', 'dev'" - type: string - default: "latest" - commit_hash: - description: "Commit hash" - required: true - -# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - CHART_NAME: litellm-helm - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - # print commit hash, tag, and release type - print: - runs-on: ubuntu-latest - steps: - - run: | - echo "Commit hash: ${{ github.event.inputs.commit_hash }}" - echo "Tag: ${{ github.event.inputs.tag }}" - echo "Release type: ${{ github.event.inputs.release_type }}" - docker-hub-deploy: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-database image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./docker/Dockerfile.database - tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} - - - name: Build and push litellm-spend-logs image - uses: docker/build-push-action@v5 - with: - context: . - push: true - file: ./litellm-js/spend-logs/Dockerfile - tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }} - - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - # - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Build and push Docker image - uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' - labels: ${{ steps.meta.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-database: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for database Dockerfile - id: meta-database - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push Database Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./docker/Dockerfile.database - push: true - tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} - labels: ${{ steps.meta-database.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-non_root: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for non_root Dockerfile - id: meta-non_root - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push non_root Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./docker/Dockerfile.non_root - push: true - tags: ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }} - labels: ${{ steps.meta-non_root.outputs.labels }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-image-spend-logs: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.commit_hash }} - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for spend-logs Dockerfile - id: meta-spend-logs - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs - # Configure multi platform Docker builds - - name: Set up QEMU - uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - - name: Build and push Database Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: ./litellm-js/spend-logs/Dockerfile - push: true - tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} - platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 - - build-and-push-helm-chart: - if: github.event.inputs.release_type != 'dev' - needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: lowercase github.repository_owner - run: | - echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - - name: Get LiteLLM Latest Tag - id: current_app_tag - shell: bash - run: | - LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0) - if [ -z "${LATEST_TAG}" ]; then - echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT - else - echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT - fi - - - name: Get last published chart version - id: current_version - shell: bash - run: | - CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true) - if [ -z "${CHART_LIST}" ]; then - echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT - else - printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT - fi - env: - HELM_EXPERIMENTAL_OCI: '1' - - # Automatically update the helm chart version one "patch" level - - name: Bump release version - id: bump_version - uses: christian-draeger/increment-semantic-version@1.1.0 - with: - current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }} - version-fragment: 'bug' - - - uses: ./.github/actions/helm-oci-chart-releaser - with: - name: ${{ env.CHART_NAME }} - repository: ${{ env.REPO_OWNER }} - tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }} - app_version: ${{ steps.current_app_tag.outputs.latest_tag }} - path: deploy/charts/${{ env.CHART_NAME }} - registry: ${{ env.REGISTRY }} - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} - update_dependencies: true - - release: - name: "New LiteLLM Release" - needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] - - runs-on: "ubuntu-latest" - - steps: - - name: Display version - run: echo "Current version is ${{ github.event.inputs.tag }}" - - name: "Set Release Tag" - run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV - - name: Display release tag - run: echo "RELEASE_TAG is $RELEASE_TAG" - - name: "Create release" - uses: "actions/github-script@v6" - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - script: | - const commitHash = "${{ github.event.inputs.commit_hash}}"; - console.log("Commit Hash:", commitHash); // Add this line for debugging - try { - const response = await github.rest.repos.createRelease({ - draft: false, - generate_release_notes: true, - target_commitish: commitHash, - name: process.env.RELEASE_TAG, - owner: context.repo.owner, - prerelease: false, - repo: context.repo.repo, - tag_name: process.env.RELEASE_TAG, - }); - - core.exportVariable('RELEASE_ID', response.data.id); - core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url); - } catch (error) { - core.setFailed(error.message); - } - - name: Fetch Release Notes - id: release-notes - uses: actions/github-script@v6 - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - script: | - try { - const response = await github.rest.repos.getRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - release_id: process.env.RELEASE_ID, - }); - const formattedBody = JSON.stringify(response.data.body).slice(1, -1); - return formattedBody; - } catch (error) { - core.setFailed(error.message); - } - env: - RELEASE_ID: ${{ env.RELEASE_ID }} - - name: Github Releases To Discord - env: - WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} - REALEASE_TAG: ${{ env.RELEASE_TAG }} - RELEASE_NOTES: ${{ steps.release-notes.outputs.result }} - run: | - curl -H "Content-Type: application/json" -X POST -d '{ - "content": "New LiteLLM release '"${RELEASE_TAG}"'", - "username": "Release Changelog", - "avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png", - "embeds": [ - { - "title": "Changelog for LiteLLM '"${RELEASE_TAG}"'", - "description": "'"${RELEASE_NOTES}"'", - "color": 2105893 - } - ] - }' $WEBHOOK_URL - diff --git a/.github/workflows/ghcr_helm_deploy.yml b/.github/workflows/ghcr_helm_deploy.yml deleted file mode 100644 index f78dc6f0f3f..00000000000 --- a/.github/workflows/ghcr_helm_deploy.yml +++ /dev/null @@ -1,67 +0,0 @@ -# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM -name: Build, Publish LiteLLM Helm Chart. New Release -on: - workflow_dispatch: - inputs: - chartVersion: - description: "Update the helm chart's version to this" - -# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - REPO_OWNER: ${{github.repository_owner}} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-helm-chart: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: lowercase github.repository_owner - run: | - echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - - name: Get LiteLLM Latest Tag - id: current_app_tag - uses: WyriHaximus/github-action-get-previous-tag@v1.3.0 - - - name: Get last published chart version - id: current_version - shell: bash - run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT - env: - HELM_EXPERIMENTAL_OCI: '1' - - # Automatically update the helm chart version one "patch" level - - name: Bump release version - id: bump_version - uses: christian-draeger/increment-semantic-version@1.1.0 - with: - current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }} - version-fragment: 'bug' - - - name: Lint helm chart - run: helm lint deploy/charts/litellm-helm - - - uses: ./.github/actions/helm-oci-chart-releaser - with: - name: litellm-helm - repository: ${{ env.REPO_OWNER }} - tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }} - app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }} - path: deploy/charts/litellm-helm - registry: ${{ env.REGISTRY }} - registry_username: ${{ github.actor }} - registry_password: ${{ secrets.GITHUB_TOKEN }} - update_dependencies: true - diff --git a/.github/workflows/helm_unit_test.yml b/.github/workflows/helm_unit_test.yml index c4b83af70a1..06836b1d1cd 100644 --- a/.github/workflows/helm_unit_test.yml +++ b/.github/workflows/helm_unit_test.yml @@ -6,22 +6,36 @@ on: branches: - main +permissions: + contents: read + jobs: unit-test: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false - name: Set up Helm 3.11.1 - uses: azure/setup-helm@v1 + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: - version: '3.11.1' + version: "3.11.1" - name: Install Helm Unit Test Plugin run: | helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 + - name: Verify Helm Unit Test Plugin integrity + run: | + EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155" + PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest" + ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)" + if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then + echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA" + exit 1 + fi + echo "Helm unittest plugin integrity verified: $ACTUAL_SHA" - name: Run unit tests - run: - helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm \ No newline at end of file + run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py deleted file mode 100644 index 6b5e6535d79..00000000000 --- a/.github/workflows/interpret_load_test.py +++ /dev/null @@ -1,138 +0,0 @@ -import csv -import os -from github import Github - - -def interpret_results(csv_file): - with open(csv_file, newline="") as csvfile: - csvreader = csv.DictReader(csvfile) - rows = list(csvreader) - """ - in this csv reader - - Create 1 new column "Status" - - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅" - - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌" - - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns - """ - - # Add a new column "Status" - for row in rows: - median_response_time = float( - row["Median Response Time"].strip().rstrip("ms") - ) - average_response_time = float( - row["Average Response Time"].strip().rstrip("s") - ) - - request_count = int(row["Request Count"]) - failure_count = int(row["Failure Count"]) - - failure_percent = round((failure_count / request_count) * 100, 2) - - # Determine status based on conditions - if ( - median_response_time < 300 - and average_response_time < 300 - and failure_percent < 5 - ): - row["Status"] = "Passed ✅" - else: - row["Status"] = "Failed ❌" - - # Construct Markdown table header - markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |" - markdown_table += ( - "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" - ) - - # Construct Markdown table rows - for row in rows: - markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |" - print("markdown table: ", markdown_table) - return markdown_table - - -def _get_docker_run_command_stable_release(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version} -``` - """ - - -def _get_docker_run_command(release_version): - return f""" -\n\n -## Docker Run LiteLLM Proxy - -``` -docker run \\ --e STORE_MODEL_IN_DB=True \\ --p 4000:4000 \\ -ghcr.io/berriai/litellm:main-{release_version} -``` - """ - - -def get_docker_run_command(release_version): - if "stable" in release_version: - return _get_docker_run_command_stable_release(release_version) - else: - return _get_docker_run_command(release_version) - - -if __name__ == "__main__": - csv_file = "load_test_stats.csv" # Change this to the path of your CSV file - markdown_table = interpret_results(csv_file) - - # Update release body with interpreted results - github_token = os.getenv("GITHUB_TOKEN") - g = Github(github_token) - repo = g.get_repo( - "BerriAI/litellm" - ) # Replace with your repository's username and name - latest_release = repo.get_latest_release() - print("got latest release: ", latest_release) - print(latest_release.title) - print(latest_release.tag_name) - - release_version = latest_release.title - - print("latest release body: ", latest_release.body) - print("markdown table: ", markdown_table) - - # check if "Load Test LiteLLM Proxy Results" exists - existing_release_body = latest_release.body - if "Load Test LiteLLM Proxy Results" in latest_release.body: - # find the "Load Test LiteLLM Proxy Results" section and delete it - start_index = latest_release.body.find("Load Test LiteLLM Proxy Results") - existing_release_body = latest_release.body[:start_index] - - docker_run_command = get_docker_run_command(release_version) - print("docker run command: ", docker_run_command) - - new_release_body = ( - existing_release_body - + docker_run_command - + "\n\n" - + "### Don't want to maintain your internal proxy? get in touch 🎉" - + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" - + "\n\n" - + "## Load Test LiteLLM Proxy Results" - + "\n\n" - + markdown_table - ) - print("new release body: ", new_release_body) - try: - latest_release.update_release( - name=latest_release.tag_name, - message=new_release_body, - ) - except Exception as e: - print(e) diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml new file mode 100644 index 00000000000..7e2693209b6 --- /dev/null +++ b/.github/workflows/issue-keyword-labeler.yml @@ -0,0 +1,65 @@ +name: Issue Keyword Labeler + +on: + issues: + types: + - opened + +jobs: + scan-and-label: + runs-on: ubuntu-latest + permissions: + issues: write + contents: read + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Scan for provider keywords + id: scan + env: + PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }} + KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek + run: python3 .github/scripts/scan_keywords.py + + - name: Ensure label exists + if: steps.scan.outputs.found == 'true' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'llm translation'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: 'c1ff72', + description: 'Issues related to LLM provider translation/mapping' + }); + } else { + throw error; + } + } + + - name: Add label to the issue + if: steps.scan.outputs.found == 'true' + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: ['llm translation'] + }); diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml new file mode 100644 index 00000000000..e0c2fa94d8c --- /dev/null +++ b/.github/workflows/label-component.yml @@ -0,0 +1,116 @@ +name: Label Component Issues + +on: + issues: + types: + - opened + +jobs: + add-component-label: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Add component labels + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const body = context.payload.issue.body; + if (!body) return; + + // Define component mappings with regex patterns that handle flexible whitespace + const components = [ + { + pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/, + label: 'sdk', + color: '0E7C86', + description: 'Issues related to the litellm Python SDK' + }, + { + pattern: /What part of LiteLLM is this about\?\s*Proxy/, + label: 'proxy', + color: '5319E7', + description: 'Issues related to the LiteLLM Proxy' + }, + { + pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/, + label: 'ui-dashboard', + color: 'D876E3', + description: 'Issues related to the LiteLLM UI Dashboard' + }, + { + pattern: /What part of LiteLLM is this about\?\s*Docs/, + label: 'docs', + color: 'FBCA04', + description: 'Issues related to LiteLLM documentation' + } + ]; + + // Find matching component + for (const component of components) { + if (component.pattern.test(body)) { + // Ensure label exists + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: component.label + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: component.label, + color: component.color, + description: component.description + }); + } + } + + // Add label to issue + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [component.label] + }); + + break; + } + } + + // Check for 'claude code' keyword (can be applied alongside component labels) + if (/claude code/i.test(body)) { + const claudeLabel = { + name: 'claude code', + color: '7c3aed', + description: 'Issues related to Claude Code usage' + }; + + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: claudeLabel.name + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: claudeLabel.name, + color: claudeLabel.color, + description: claudeLabel.description + }); + } + } + + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [claudeLabel.name] + }); + } diff --git a/.github/workflows/label-mlops.yml b/.github/workflows/label-mlops.yml deleted file mode 100644 index 37789c1ea76..00000000000 --- a/.github/workflows/label-mlops.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Label ML Ops Team Issues - -on: - issues: - types: - - opened - -jobs: - add-mlops-label: - runs-on: ubuntu-latest - steps: - - name: Check if ML Ops Team is selected - uses: actions-ecosystem/action-add-labels@v1 - if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes') - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - labels: "mlops user request" diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml new file mode 100644 index 00000000000..922013c4b54 --- /dev/null +++ b/.github/workflows/llm-translation-testing.yml @@ -0,0 +1,93 @@ +name: LLM Translation Tests + +on: + workflow_dispatch: + inputs: + release_candidate_tag: + description: "Release candidate tag/version" + required: true + type: string + push: + tags: + - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1 + +permissions: + contents: read + +jobs: + run-llm-translation-tests: + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + ref: ${{ github.event.inputs.release_candidate_tag || github.ref }} + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Install Poetry + run: | + pip install 'poetry==2.3.2' + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + + - name: Restore Poetry dependencies cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 + with: + path: | + ~/.cache/pypoetry + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry install --with dev + poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0' + + - name: Create test results directory + run: mkdir -p test-results + + - name: Run LLM Translation Tests + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} + AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} + AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} + RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }} + COMMIT_SHA: ${{ github.sha }} + run: | + python .github/workflows/run_llm_translation_tests.py \ + --tag "$RC_TAG" \ + --commit "$COMMIT_SHA" \ + || true # Continue even if tests fail + + - name: Display test summary + if: always() + run: | + if [ -f "test-results/llm_translation_report.md" ]; then + echo "Test report generated successfully!" + echo "Artifact will contain:" + echo "- test-results/junit.xml (JUnit XML results)" + echo "- test-results/llm_translation_report.md (Beautiful markdown report)" + else + echo "Warning: Test report was not generated" + fi + + - name: Upload test artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }} + path: test-results/ + retention-days: 30 diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml deleted file mode 100644 index cdaffa328c9..00000000000 --- a/.github/workflows/load_test.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Test Locust Load Test - -on: - workflow_run: - workflows: ["Build, Publish LiteLLM Docker Image. New Release"] - types: - - completed - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install PyGithub - - name: re-deploy proxy - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/redeploy_proxy.py" - env: - LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }} - LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }} - working-directory: ${{ github.workspace }} - - name: Run Load Test - id: locust_run - uses: BerriAI/locust-github-action@master - with: - LOCUSTFILE: ".github/workflows/locustfile.py" - URL: "https://post-release-load-test-proxy.onrender.com/" - USERS: "20" - RATE: "20" - RUNTIME: "300s" - - name: Process Load Test Stats - run: | - echo "Current working directory: $PWD" - ls - python ".github/workflows/interpret_load_test.py" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: ${{ github.workspace }} - - name: Upload CSV as Asset to Latest Release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - file: "load_test_stats.csv;load_test.html" - update_latest_release: true - tag_name: "load-test" - overwrite: true \ No newline at end of file diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py deleted file mode 100644 index 36dbeee9c48..00000000000 --- a/.github/workflows/locustfile.py +++ /dev/null @@ -1,28 +0,0 @@ -from locust import HttpUser, task, between - - -class MyUser(HttpUser): - wait_time = between(1, 5) - - @task - def chat_completion(self): - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg", - # Include any additional headers you may need for authentication, etc. - } - - # Customize the payload with "model" and "messages" keys - payload = { - "model": "fake-openai-endpoint", - "messages": [ - {"role": "system", "content": "You are a chat bot."}, - {"role": "user", "content": "Hello, how are you?"}, - ], - # Add more data as necessary - } - - # Make a POST request to the "chat/completions" endpoint - response = self.client.post("chat/completions", json=payload, headers=headers) - - # Print or log the response if needed diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 23e4a06da9e..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Publish Dev Release to PyPI - -on: - workflow_dispatch: - -jobs: - publish-dev-release: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml twine - - - name: Read version from pyproject.toml - id: read-version - run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') - printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - - - name: Check if version exists on PyPI - id: check-version - run: | - set -e - if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then - echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish." - diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml new file mode 100644 index 00000000000..8f675bb3075 --- /dev/null +++ b/.github/workflows/publish_to_pypi.yml @@ -0,0 +1,136 @@ +name: Publish to PyPI + +on: + workflow_dispatch: + +jobs: + preflight-checks: + name: Preflight Checks + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + # No environment — read-only checks, no approval needed + outputs: + needs_publish: ${{ steps.check-litellm.outputs.needs_publish }} + version: ${{ steps.check-litellm.outputs.version }} + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Check litellm version on PyPI + id: check-litellm + run: | + VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Checking if litellm $VERSION exists on PyPI..." + + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json") + if [ "$HTTP_STATUS" = "200" ]; then + echo "litellm $VERSION already exists on PyPI. Skipping publish." + echo "needs_publish=false" >> "$GITHUB_OUTPUT" + else + echo "litellm $VERSION not found on PyPI. Publish needed." + echo "needs_publish=true" >> "$GITHUB_OUTPUT" + fi + + - name: Sanity check proxy-extras version + run: | + # Read pinned version from requirements.txt + REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt) + if [ -z "$REQ_VERSION" ]; then + echo "::error::Could not find litellm-proxy-extras version in requirements.txt" + exit 1 + fi + echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION" + + # Read pinned version from pyproject.toml dependency + PYPROJECT_VERSION=$(python3 -c " + import re + with open('pyproject.toml') as f: + content = f.read() + match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content) + if match: + print(match.group(1).lstrip('^~>=')) + else: + import sys + print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr) + sys.exit(1) + ") + echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION" + + # Check that both pinned versions match + if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then + echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION" + exit 1 + fi + + # Check that the pinned version exists on PyPI + echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..." + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json") + if [ "$HTTP_STATUS" != "200" ]; then + echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm." + exit 1 + fi + echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed." + + publish-litellm: + name: Publish litellm to PyPI + needs: preflight-checks + if: needs.preflight-checks.outputs.needs_publish == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + id-token: write + contents: read + environment: pypi-publish + + steps: + - name: Checkout repo + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Copy model prices backup + run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json + + - name: Install build tools + run: python -m pip install --upgrade pip build==1.4.2 + + - name: Build package + run: | + rm -rf build dist + python -m build + + - name: Verify build artifacts + env: + EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }} + run: | + echo "Contents of dist/:" + ls -la dist/ + # Ensure we have both sdist and wheel + ls dist/*.tar.gz + ls dist/*.whl + # Verify built version matches expected + ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || { + echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION" + ls dist/ + exit 1 + } + + - name: Validate package metadata + run: | + pip install twine==6.2.0 + twine check dist/* + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/read_pyproject_version.yml b/.github/workflows/read_pyproject_version.yml index 8f6310f935b..04b4a38ce19 100644 --- a/.github/workflows/read_pyproject_version.yml +++ b/.github/workflows/read_pyproject_version.yml @@ -3,7 +3,10 @@ name: Read Version from pyproject.toml on: push: branches: - - main # Change this to the default branch of your repository + - main # Change this to the default branch of your repository + +permissions: + contents: read jobs: read-version: @@ -11,20 +14,14 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - python-version: 3.8 # Adjust the Python version as needed - - - name: Install dependencies - run: pip install toml + persist-credentials: false - name: Read version from pyproject.toml id: read-version run: | - version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])') + version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/') printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV - name: Display version diff --git a/.github/workflows/redeploy_proxy.py b/.github/workflows/redeploy_proxy.py deleted file mode 100644 index ed46bef73a2..00000000000 --- a/.github/workflows/redeploy_proxy.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - -redeploy_proxy.py -""" - -import os -import requests -import time - -# send a get request to this endpoint -deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1") -response = requests.get(deploy_hook1, timeout=20) - - -deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2") -response = requests.get(deploy_hook2, timeout=20) - -print("SENT GET REQUESTS to re-deploy proxy") -print("sleeeping.... for 60s") -time.sleep(60) diff --git a/.github/workflows/reset_stable.yml b/.github/workflows/reset_stable.yml deleted file mode 100644 index f6fed672d47..00000000000 --- a/.github/workflows/reset_stable.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Reset litellm_stable branch - -on: - release: - types: [published, created] -jobs: - update-stable-branch: - if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }} - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Reset litellm_stable_release_branch branch to the release commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Configure Git user - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Fetch all branches and tags - git fetch --all - - # Check if the litellm_stable_release_branch branch exists - if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then - echo "litellm_stable_release_branch branch exists." - git checkout litellm_stable_release_branch - else - echo "litellm_stable_release_branch branch does not exist. Creating it." - git checkout -b litellm_stable_release_branch - fi - - # Reset litellm_stable_release_branch branch to the release commit - git reset --hard $GITHUB_SHA - - # Push the updated litellm_stable_release_branch branch - git push origin litellm_stable_release_branch --force diff --git a/.github/workflows/run_llm_translation_tests.py b/.github/workflows/run_llm_translation_tests.py new file mode 100644 index 00000000000..5b3a4817ecb --- /dev/null +++ b/.github/workflows/run_llm_translation_tests.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +""" +Run LLM Translation Tests and Generate Beautiful Markdown Report + +This script runs the LLM translation tests and generates a comprehensive +markdown report with provider-specific breakdowns and test statistics. +""" + +import os +import sys +import subprocess +import xml.etree.ElementTree as ET +from collections import defaultdict +from datetime import datetime +from pathlib import Path +import json +from typing import Dict, List, Tuple, Optional + +# ANSI color codes for terminal output +class Colors: + GREEN = '\033[92m' + RED = '\033[91m' + YELLOW = '\033[93m' + BLUE = '\033[94m' + PURPLE = '\033[95m' + CYAN = '\033[96m' + RESET = '\033[0m' + BOLD = '\033[1m' + +def print_colored(message: str, color: str = Colors.RESET): + """Print colored message to terminal""" + print(f"{color}{message}{Colors.RESET}") + +def get_provider_from_test_file(test_file: str) -> str: + """Map test file names to provider names""" + provider_mapping = { + 'test_anthropic': 'Anthropic', + 'test_azure': 'Azure', + 'test_bedrock': 'AWS Bedrock', + 'test_openai': 'OpenAI', + 'test_vertex': 'Google Vertex AI', + 'test_gemini': 'Google Vertex AI', + 'test_cohere': 'Cohere', + 'test_databricks': 'Databricks', + 'test_groq': 'Groq', + 'test_together': 'Together AI', + 'test_mistral': 'Mistral', + 'test_deepseek': 'DeepSeek', + 'test_replicate': 'Replicate', + 'test_huggingface': 'HuggingFace', + 'test_fireworks': 'Fireworks AI', + 'test_perplexity': 'Perplexity', + 'test_cloudflare': 'Cloudflare', + 'test_voyage': 'Voyage AI', + 'test_xai': 'xAI', + 'test_nvidia': 'NVIDIA', + 'test_watsonx': 'IBM watsonx', + 'test_azure_ai': 'Azure AI', + 'test_snowflake': 'Snowflake', + 'test_infinity': 'Infinity', + 'test_jina': 'Jina AI', + 'test_deepgram': 'Deepgram', + 'test_clarifai': 'Clarifai', + 'test_triton': 'Triton', + } + + for key, provider in provider_mapping.items(): + if key in test_file: + return provider + + # For cross-provider test files + if any(name in test_file for name in ['test_optional_params', 'test_prompt_factory', + 'test_router', 'test_text_completion']): + return f'Cross-Provider Tests ({test_file})' + + return 'Other Tests' + +def format_duration(seconds: float) -> str: + """Format duration in human-readable format""" + if seconds < 60: + return f"{seconds:.2f}s" + elif seconds < 3600: + minutes = int(seconds // 60) + secs = seconds % 60 + return f"{minutes}m {secs:.0f}s" + else: + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + return f"{hours}h {minutes}m" + + +def generate_markdown_report(junit_xml_path: str, output_path: str, tag: str = None, commit: str = None): + """Generate a beautiful markdown report from JUnit XML""" + try: + tree = ET.parse(junit_xml_path) + root = tree.getroot() + + # Handle both testsuite and testsuites root + if root.tag == 'testsuites': + suites = root.findall('testsuite') + else: + suites = [root] + + # Overall statistics + total_tests = 0 + total_failures = 0 + total_errors = 0 + total_skipped = 0 + total_time = 0.0 + + # Provider breakdown + provider_stats = defaultdict(lambda: {'passed': 0, 'failed': 0, 'skipped': 0, 'errors': 0, 'time': 0.0}) + provider_tests = defaultdict(list) + + for suite in suites: + total_tests += int(suite.get('tests', 0)) + total_failures += int(suite.get('failures', 0)) + total_errors += int(suite.get('errors', 0)) + total_skipped += int(suite.get('skipped', 0)) + total_time += float(suite.get('time', 0)) + + for testcase in suite.findall('testcase'): + classname = testcase.get('classname', '') + test_name = testcase.get('name', '') + test_time = float(testcase.get('time', 0)) + + # Extract test file name from classname + if '.' in classname: + parts = classname.split('.') + test_file = parts[-2] if len(parts) > 1 else 'unknown' + else: + test_file = 'unknown' + + provider = get_provider_from_test_file(test_file) + provider_stats[provider]['time'] += test_time + + # Check test status + if testcase.find('failure') is not None: + provider_stats[provider]['failed'] += 1 + failure = testcase.find('failure') + failure_msg = failure.get('message', '') if failure is not None else '' + provider_tests[provider].append({ + 'name': test_name, + 'status': 'FAILED', + 'time': test_time, + 'message': failure_msg + }) + elif testcase.find('error') is not None: + provider_stats[provider]['errors'] += 1 + error = testcase.find('error') + error_msg = error.get('message', '') if error is not None else '' + provider_tests[provider].append({ + 'name': test_name, + 'status': 'ERROR', + 'time': test_time, + 'message': error_msg + }) + elif testcase.find('skipped') is not None: + provider_stats[provider]['skipped'] += 1 + skip = testcase.find('skipped') + skip_msg = skip.get('message', '') if skip is not None else '' + provider_tests[provider].append({ + 'name': test_name, + 'status': 'SKIPPED', + 'time': test_time, + 'message': skip_msg + }) + else: + provider_stats[provider]['passed'] += 1 + provider_tests[provider].append({ + 'name': test_name, + 'status': 'PASSED', + 'time': test_time, + 'message': '' + }) + + passed = total_tests - total_failures - total_errors - total_skipped + + # Generate the markdown report + with open(output_path, 'w') as f: + # Header + f.write("# LLM Translation Test Results\n\n") + + # Metadata table + f.write("## Test Run Information\n\n") + f.write("| Field | Value |\n") + f.write("|-------|-------|\n") + f.write(f"| **Tag** | `{tag or 'N/A'}` |\n") + f.write(f"| **Date** | {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')} |\n") + f.write(f"| **Commit** | `{commit or 'N/A'}` |\n") + f.write(f"| **Duration** | {format_duration(total_time)} |\n") + f.write("\n") + + # Overall statistics with visual elements + f.write("## Overall Statistics\n\n") + + # Summary box + f.write("```\n") + f.write(f"Total Tests: {total_tests}\n") + f.write(f"├── Passed: {passed:>4} ({(passed/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n") + f.write(f"├── Failed: {total_failures:>4} ({(total_failures/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n") + f.write(f"├── Errors: {total_errors:>4} ({(total_errors/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n") + f.write(f"└── Skipped: {total_skipped:>4} ({(total_skipped/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n") + f.write("```\n\n") + + + # Provider summary table + f.write("## Results by Provider\n\n") + f.write("| Provider | Total | Pass | Fail | Error | Skip | Pass Rate | Duration |\n") + f.write("|----------|-------|------|------|-------|------|-----------|----------|") + + # Sort providers: specific providers first, then cross-provider tests + sorted_providers = [] + cross_provider = [] + for p in sorted(provider_stats.keys()): + if 'Cross-Provider' in p or p == 'Other Tests': + cross_provider.append(p) + else: + sorted_providers.append(p) + + all_providers = sorted_providers + cross_provider + + for provider in all_providers: + stats = provider_stats[provider] + total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped'] + pass_rate = (stats['passed'] / total * 100) if total > 0 else 0 + + f.write(f"\n| {provider} | {total} | {stats['passed']} | {stats['failed']} | ") + f.write(f"{stats['errors']} | {stats['skipped']} | {pass_rate:.1f}% | ") + f.write(f"{format_duration(stats['time'])} |") + + # Detailed test results by provider + f.write("\n\n## Detailed Test Results\n\n") + + for provider in sorted_providers: + if provider_tests[provider]: + stats = provider_stats[provider] + total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped'] + + f.write(f"### {provider}\n\n") + f.write(f"**Summary:** {stats['passed']}/{total} passed ") + f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%) ") + f.write(f"in {format_duration(stats['time'])}\n\n") + + # Group tests by status + tests_by_status = defaultdict(list) + for test in provider_tests[provider]: + tests_by_status[test['status']].append(test) + + # Show failed tests first (if any) + if tests_by_status['FAILED']: + f.write("
\nFailed Tests\n\n") + for test in tests_by_status['FAILED']: + f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n") + if test['message']: + # Truncate long error messages + msg = test['message'][:200] + '...' if len(test['message']) > 200 else test['message'] + f.write(f" > {msg}\n") + f.write("\n
\n\n") + + # Show errors (if any) + if tests_by_status['ERROR']: + f.write("
\nError Tests\n\n") + for test in tests_by_status['ERROR']: + f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n") + f.write("\n
\n\n") + + # Show passed tests in collapsible section + if tests_by_status['PASSED']: + f.write("
\nPassed Tests\n\n") + for test in tests_by_status['PASSED']: + f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n") + f.write("\n
\n\n") + + # Show skipped tests (if any) + if tests_by_status['SKIPPED']: + f.write("
\nSkipped Tests\n\n") + for test in tests_by_status['SKIPPED']: + f.write(f"- `{test['name']}`\n") + f.write("\n
\n\n") + + # Cross-provider tests in a separate section + if cross_provider: + f.write("### Cross-Provider Tests\n\n") + for provider in cross_provider: + if provider_tests[provider]: + stats = provider_stats[provider] + total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped'] + + f.write(f"#### {provider}\n\n") + f.write(f"**Summary:** {stats['passed']}/{total} passed ") + f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%)\n\n") + + # For cross-provider tests, just show counts + f.write(f"- Passed: {stats['passed']}\n") + if stats['failed'] > 0: + f.write(f"- Failed: {stats['failed']}\n") + if stats['errors'] > 0: + f.write(f"- Errors: {stats['errors']}\n") + if stats['skipped'] > 0: + f.write(f"- Skipped: {stats['skipped']}\n") + f.write("\n") + + + print_colored(f"Report generated: {output_path}", Colors.GREEN) + + except Exception as e: + print_colored(f"Error generating report: {e}", Colors.RED) + raise + +def run_tests(test_path: str = "tests/llm_translation/", + junit_xml: str = "test-results/junit.xml", + report_path: str = "test-results/llm_translation_report.md", + tag: str = None, + commit: str = None) -> int: + """Run the LLM translation tests and generate report""" + + # Create test results directory + os.makedirs(os.path.dirname(junit_xml), exist_ok=True) + + print_colored("Starting LLM Translation Tests", Colors.BOLD + Colors.BLUE) + print_colored(f"Test directory: {test_path}", Colors.CYAN) + print_colored(f"Output: {junit_xml}", Colors.CYAN) + print() + + # Run pytest + cmd = [ + "poetry", "run", "pytest", test_path, + f"--junitxml={junit_xml}", + "-v", + "--tb=short", + "--maxfail=500", + "-n", "auto" + ] + + # Add timeout if pytest-timeout is installed + try: + subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"], + capture_output=True, check=True) + cmd.extend(["--timeout=300"]) + except: + print_colored("Warning: pytest-timeout not installed, skipping timeout option", Colors.YELLOW) + + print_colored("Running pytest with command:", Colors.YELLOW) + print(f" {' '.join(cmd)}") + print() + + # Run the tests + result = subprocess.run(cmd, capture_output=False) + + # Generate the report regardless of test outcome + if os.path.exists(junit_xml): + print() + print_colored("Generating test report...", Colors.BLUE) + generate_markdown_report(junit_xml, report_path, tag, commit) + + # Print summary to console + print() + print_colored("Test Summary:", Colors.BOLD + Colors.PURPLE) + + # Parse XML for quick summary + tree = ET.parse(junit_xml) + root = tree.getroot() + + if root.tag == 'testsuites': + suites = root.findall('testsuite') + else: + suites = [root] + + total = sum(int(s.get('tests', 0)) for s in suites) + failures = sum(int(s.get('failures', 0)) for s in suites) + errors = sum(int(s.get('errors', 0)) for s in suites) + skipped = sum(int(s.get('skipped', 0)) for s in suites) + passed = total - failures - errors - skipped + + print(f" Total: {total}") + print_colored(f" Passed: {passed}", Colors.GREEN) + if failures > 0: + print_colored(f" Failed: {failures}", Colors.RED) + if errors > 0: + print_colored(f" Errors: {errors}", Colors.RED) + if skipped > 0: + print_colored(f" Skipped: {skipped}", Colors.YELLOW) + + if total > 0: + pass_rate = (passed / total) * 100 + color = Colors.GREEN if pass_rate >= 80 else Colors.YELLOW if pass_rate >= 60 else Colors.RED + print_colored(f" Pass Rate: {pass_rate:.1f}%", color) + else: + print_colored("No test results found!", Colors.RED) + + print() + print_colored("Test run complete!", Colors.BOLD + Colors.GREEN) + + return result.returncode + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Run LLM Translation Tests") + parser.add_argument("--test-path", default="tests/llm_translation/", + help="Path to test directory") + parser.add_argument("--junit-xml", default="test-results/junit.xml", + help="Path for JUnit XML output") + parser.add_argument("--report", default="test-results/llm_translation_report.md", + help="Path for markdown report") + parser.add_argument("--tag", help="Git tag or version") + parser.add_argument("--commit", help="Git commit SHA") + + args = parser.parse_args() + + # Get git info if not provided + if not args.commit: + try: + result = subprocess.run(["git", "rev-parse", "HEAD"], + capture_output=True, text=True) + if result.returncode == 0: + args.commit = result.stdout.strip() + except: + pass + + if not args.tag: + try: + result = subprocess.run(["git", "describe", "--tags", "--abbrev=0"], + capture_output=True, text=True) + if result.returncode == 0: + args.tag = result.stdout.strip() + except: + pass + + exit_code = run_tests( + test_path=args.test_path, + junit_xml=args.junit_xml, + report_path=args.report, + tag=args.tag, + commit=args.commit + ) + + sys.exit(exit_code) \ No newline at end of file diff --git a/.github/workflows/run_observatory_tests.yml b/.github/workflows/run_observatory_tests.yml new file mode 100644 index 00000000000..a25b96766d7 --- /dev/null +++ b/.github/workflows/run_observatory_tests.yml @@ -0,0 +1,229 @@ +name: Run Observatory Tests +on: + workflow_dispatch: + inputs: + tag: + description: "Docker image tag to test (e.g. v1.61.0.rc1)" + required: true + type: string + commit_hash: + description: "Commit hash (defaults to HEAD of current branch)" + required: false + type: string + workflow_call: + inputs: + tag: + description: "Docker image tag to test" + required: true + type: string + commit_hash: + description: "Commit hash of the release" + required: true + type: string + +permissions: + contents: read + +env: + LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }} + +jobs: + observatory-tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Validate tag input + env: + TAG: ${{ inputs.tag }} + run: | + if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "Invalid tag format: $TAG (expected vX.Y.Z...)" + exit 1 + fi + + - name: Start LiteLLM container + env: + TAG: ${{ inputs.tag }} + AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} + AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }} + WORKSPACE: ${{ github.workspace }} + run: | + docker run -d \ + --name litellm-rc \ + -p 4000:4000 \ + -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \ + -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ + -e AZURE_API_KEY="${AZURE_API_KEY}" \ + -e AZURE_API_BASE="${AZURE_API_BASE}" \ + "litellm/litellm:${TAG}" \ + --config /app/config.yaml --port 4000 + + - name: Wait for LiteLLM health check + run: | + echo "Waiting for LiteLLM to be ready..." + for i in $(seq 1 30); do + if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then + echo "LiteLLM is healthy" + exit 0 + fi + echo "Attempt $i/30 - not ready yet, waiting 10s..." + sleep 10 + done + echo "LiteLLM failed to start within 5 minutes" + docker logs litellm-rc + exit 1 + + - name: Start cloudflared tunnel + run: | + # Install cloudflared (pinned version + checksum) + curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared + echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c - + chmod +x /usr/local/bin/cloudflared + + # Start a quick tunnel (no account needed) and capture the URL + cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 & + CLOUDFLARED_PID=$! + echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV + + # Wait for tunnel URL to appear in logs + echo "Waiting for tunnel URL..." + for i in $(seq 1 30); do + TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true) + if [ -n "$TUNNEL_URL" ]; then + echo "Tunnel URL: $TUNNEL_URL" + echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV + exit 0 + fi + sleep 2 + done + echo "Failed to get tunnel URL" + cat /tmp/cloudflared.log + exit 1 + + - name: Verify tunnel connectivity + run: | + echo "Testing tunnel at ${TUNNEL_URL}..." + # Quick tunnels need time for DNS propagation; retry to avoid + # transient NXDOMAIN (curl exit code 6) on first attempt. + for i in $(seq 1 10); do + if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then + echo "Tunnel is working (attempt $i)" + exit 0 + fi + echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..." + sleep 5 + done + echo "Tunnel failed to become reachable after 50s" + cat /tmp/cloudflared.log + exit 1 + + - name: Trigger observatory test run + id: trigger + env: + OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }} + OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }} + run: | + PAYLOAD=$(jq -n \ + --arg url "${TUNNEL_URL}" \ + --arg key "${LITELLM_MASTER_KEY}" \ + '{ + deployment_url: $url, + api_key: $key, + test_suite: "TestOAIAzureRelease", + models: ["gpt-4o-mini", "gpt-4o"] + }') + RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \ + -H "Content-Type: application/json" \ + -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \ + -d "$PAYLOAD") + HTTP_CODE=$(echo "$RESPONSE" | tail -1) + BODY=$(echo "$RESPONSE" | head -n -1) + echo "Response ($HTTP_CODE): $BODY" + if [ "$HTTP_CODE" -ge 400 ]; then + echo "Failed to trigger test run" + exit 1 + fi + + # Extract request_id for polling this specific run + REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id') + if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then + echo "Failed to extract request_id from response" + exit 1 + fi + echo "Request ID: $REQUEST_ID" + echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT + + - name: Poll for test completion + id: poll + env: + OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }} + OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }} + REQUEST_ID: ${{ steps.trigger.outputs.request_id }} + run: | + TIMEOUT=900 # 15 minutes + INTERVAL=30 + ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \ + -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}") + RUN_STATUS=$(echo "$STATUS" | jq -r '.status') + echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS" + + if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then + echo "Test finished with status: $RUN_STATUS" + echo "$STATUS" > /tmp/observatory_result.json + exit 0 + fi + + sleep $INTERVAL + ELAPSED=$((ELAPSED + INTERVAL)) + done + echo "Timed out waiting for test to complete after ${TIMEOUT}s" + exit 1 + + - name: Verify test results + run: | + RESULT=$(cat /tmp/observatory_result.json) + echo "Full result: $RESULT" + + STATUS=$(echo "$RESULT" | jq -r '.status') + TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false') + FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"') + ERROR=$(echo "$RESULT" | jq -r '.error // empty') + + echo "Status: $STATUS" + echo "Test passed: $TEST_PASSED" + echo "Failure rate: $FAILURE_RATE" + + if [ -n "$ERROR" ]; then + echo "Error: $ERROR" + fi + + if [ "$STATUS" = "failed" ]; then + echo "Test run failed" + exit 1 + fi + + if [ "$TEST_PASSED" != "true" ]; then + echo "Tests did not pass (failure rate: $FAILURE_RATE)" + exit 1 + fi + + echo "All tests passed!" + + - name: Print LiteLLM logs on failure + if: failure() + run: | + docker logs litellm-rc 2>/dev/null || true + cat /tmp/cloudflared.log 2>/dev/null || true + + - name: Cleanup + if: always() + run: | + kill "$CLOUDFLARED_PID" 2>/dev/null || true + docker rm -f litellm-rc 2>/dev/null || true diff --git a/.github/workflows/scan_duplicate_issues.yml b/.github/workflows/scan_duplicate_issues.yml new file mode 100644 index 00000000000..222ff11f304 --- /dev/null +++ b/.github/workflows/scan_duplicate_issues.yml @@ -0,0 +1,48 @@ +name: Scan Duplicate Issues (One-Time) + +on: + workflow_dispatch: + inputs: + threshold: + description: "Similarity threshold (0-1)" + required: false + default: "0.85" + close: + description: "Actually close duplicates (false = dry run)" + required: false + type: boolean + default: false + +jobs: + scan: + runs-on: ubuntu-latest + permissions: + issues: write + contents: read + steps: + - name: Checkout scripts + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + + - name: Scan for duplicate issues + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUT_THRESHOLD: ${{ inputs.threshold }} + INPUT_CLOSE: ${{ inputs.close }} + run: | + CLOSE_FLAG="" + if [ "$INPUT_CLOSE" = "true" ]; then + CLOSE_FLAG="--close" + fi + python3 .github/scripts/close_duplicate_issues.py \ + --scan \ + --repo ${{ github.repository }} \ + --threshold "$INPUT_THRESHOLD" \ + $CLOSE_FLAG diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000000..3a00064c3bd --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,47 @@ +name: Scorecard supply-chain security + +on: + branch_protection_rule: + schedule: + - cron: '27 12 * * 4' + push: + branches: ["main"] + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + if: github.event.repository.default_branch == github.ref_name + permissions: + security-events: write + id-token: write + # Uncomment for private repos if needed: + # contents: read + # actions: read + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + - name: Upload to code scanning + uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1 + with: + sarif_file: results.sarif diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5a9b19fc9ca..c905bb12312 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,19 +2,24 @@ name: "Stale Issue Management" on: schedule: - - cron: '0 0 * * *' # Runs daily at midnight UTC + - cron: "0 0 * * *" # Runs daily at midnight UTC workflow_dispatch: +permissions: + issues: write + pull-requests: write + jobs: stale: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs." - days-before-stale: 90 # Revert to 60 days - days-before-close: 7 # Revert to 7 days + days-before-stale: 90 # Revert to 60 days + days-before-close: 7 # Revert to 7 days stale-issue-label: "stale" - operations-per-run: 1000 \ No newline at end of file + operations-per-run: 1000 diff --git a/.github/workflows/sync-schema.yml b/.github/workflows/sync-schema.yml new file mode 100644 index 00000000000..72a5c56293e --- /dev/null +++ b/.github/workflows/sync-schema.yml @@ -0,0 +1,73 @@ +name: Sync schema.prisma copies + +on: + pull_request: + paths: + - 'schema.prisma' + +# Scoped to ONLY the permissions needed: +# - contents:write to push the sync commit to the PR branch +# - pull-requests:read is implicit (needed to check out the PR) +permissions: + contents: write + +jobs: + sync: + name: Copy root schema to proxy and proxy-extras + runs-on: ubuntu-latest + timeout-minutes: 5 + # Only run on PRs from branches in THIS repo (not forks). + # Fork PRs cannot push back to the head branch with GITHUB_TOKEN, + # and pull_request events from forks have read-only tokens anyway. + # Also reject PRs from branches named after protected branches to + # prevent pushing directly to main/master. + if: >- + github.event.pull_request.head.repo.full_name == github.repository + && github.head_ref != 'main' + && github.head_ref != 'master' + steps: + - name: Checkout PR branch by SHA + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + # Use the merge commit SHA for safety — github.head_ref is an + # attacker-controlled string (the branch name) and could contain + # unusual characters that cause unexpected git behavior. + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: true # needed for git push + + - name: Reject symlinked schema files + run: | + for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do + if [ -L "$f" ]; then + echo "::error file=$f::$f is a symlink, which is not allowed" + exit 1 + fi + done + + - name: Copy root schema to other locations + run: | + cp schema.prisma litellm/proxy/schema.prisma + cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + + - name: Check for changes + id: diff + run: | + if git diff --quiet -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Schemas already in sync. Nothing to do." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "Schema copies need updating." + fi + + - name: Commit synced schemas + if: steps.diff.outputs.changed == 'true' + run: | + # Push to the PR's head branch (need the branch name for git push). + # We checked out by SHA above for safety, so configure the push target explicitly. + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$GITHUB_HEAD_REF" + git add -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma + git commit -m "chore: sync schema.prisma copies from root" + git push origin "HEAD:$GITHUB_HEAD_REF" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml new file mode 100644 index 00000000000..5bb85716a17 --- /dev/null +++ b/.github/workflows/test-linting.yml @@ -0,0 +1,115 @@ +name: LiteLLM Linting + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + clean: true + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Clean Python cache + run: | + find . -type d -name "__pycache__" -exec rm -rf {} + || true + find . -name "*.pyc" -delete || true + + - name: Check poetry.lock is up to date + run: | + poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) + + - name: Install dependencies + run: | + poetry install --with dev + + - name: Check Black formatting + run: | + cd litellm + poetry run black --check --exclude '/enterprise/' . + cd .. + + - name: Debug - Check file state + run: | + echo "Current branch:" + git branch --show-current + echo "Last 3 commits:" + git log --oneline -3 + echo "File content around line 43:" + head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 + + - name: Run Ruff linting + run: | + cd litellm + poetry run ruff check . + cd .. + + - name: Print OpenAI version + run: | + poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')" + + - name: Run MyPy type checking + run: | + cd litellm + poetry run mypy . + cd .. + + - name: Check for circular imports + run: | + cd litellm + poetry run python ../tests/documentation_tests/test_circular_imports.py + cd .. + + - name: Check import safety + run: | + poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) + + secret-scan: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Run secret scan test + run: | + pip install 'pytest==9.0.2' + pytest tests/litellm/test_no_hardcoded_secrets.py -v + + - name: Run ggshield secret scan + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + run: | + if [ -n "$GITGUARDIAN_API_KEY" ]; then + pip install 'ggshield==1.48.0' + ggshield secret scan repo . + else + echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan" + fi diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml new file mode 100644 index 00000000000..dafabe4d83e --- /dev/null +++ b/.github/workflows/test-litellm-matrix.yml @@ -0,0 +1,214 @@ +name: LiteLLM Unit Tests (Matrix) + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +# Cancel in-progress runs for the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 # Increased from 15 to 20 + strategy: + fail-fast: false + matrix: + test-group: + # tests/test_litellm split by subdirectory (~560 files total) + # Vertex AI tests separated for better isolation (prevent auth/env pollution) + - name: "llms-vertex" + path: "tests/test_litellm/llms/vertex_ai" + workers: 1 + reruns: 2 + - name: "llms-other" + path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + workers: 2 + reruns: 2 + # tests/test_litellm/proxy split by subdirectory (~180 files total) + - name: "proxy-guardrails" + path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" + workers: 2 + reruns: 2 + - name: "proxy-core" + path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" + workers: 2 + reruns: 2 + - name: "proxy-misc" + path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" + workers: 2 + reruns: 2 + - name: "integrations" + path: "tests/test_litellm/integrations" + workers: 2 + reruns: 3 # Integration tests tend to be flakier + - name: "core-utils" + path: "tests/test_litellm/litellm_core_utils" + workers: 2 + reruns: 1 + - name: "other-1" + # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines + path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" + workers: 2 + reruns: 2 + - name: "other-2" + # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines + path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils" + workers: 2 + reruns: 2 + - name: "other-3" + # remaining dirs ≈ 8.0k lines + path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores" + workers: 2 + reruns: 2 + - name: "root" + path: "tests/test_litellm/test_*.py" + workers: 2 + reruns: 2 + # tests/proxy_unit_tests split alphabetically (~48 files total) + - name: "proxy-unit-a1" + # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest + path: "tests/proxy_unit_tests/test_[a-j]*.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-a2" + # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback + path: "tests/proxy_unit_tests/test_[k-o]*.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b1" + # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*) + path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b2" + # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests + path: "tests/proxy_unit_tests/test_proxy_server.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b3" + # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests + path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b4" + # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter + path: "tests/proxy_unit_tests/test_proxy_utils.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b5" + # proxy_token_counter (1279) - runs independently from utils + path: "tests/proxy_unit_tests/test_proxy_token_counter.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b6" + # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62) + path: "tests/proxy_unit_tests/test_[r-t]*.py" + workers: 2 + reruns: 1 + - name: "proxy-unit-b7" + # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157) + path: "tests/proxy_unit_tests/test_[u-z]*.py" + workers: 2 + reruns: 1 + + name: test (${{ matrix.test-group.name }}) + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests - ${{ matrix.test-group.name }} + run: | + poetry run pytest ${{ matrix.test-group.path }} \ + --tb=short -vv \ + --maxfail=10 \ + -n ${{ matrix.test-group.workers }} \ + --reruns ${{ matrix.test-group.reruns }} \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 \ + --cov=litellm \ + --cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \ + --cov-config=pyproject.toml + + - name: Save coverage report + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: coverage-${{ matrix.test-group.name }} + path: coverage-${{ matrix.test-group.name }}.xml + retention-days: 1 + + upload-coverage: + name: Upload coverage to Codecov + needs: test + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # Required for OIDC tokenless upload + pull-requests: write # Required for Codecov PR comments + + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Download all coverage reports + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 + with: + pattern: coverage-* + path: coverage-reports + merge-multiple: true + + - name: Upload to Codecov + uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4 + with: + use_oidc: true + directory: coverage-reports + root_dir: ${{ github.workspace }} + fail_ci_if_error: false diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml new file mode 100644 index 00000000000..bef568298e0 --- /dev/null +++ b/.github/workflows/test-litellm-ui-build.yml @@ -0,0 +1,34 @@ +name: UI Build Check +permissions: + contents: read + +on: + pull_request: + branches: [main] + +jobs: + build-ui: + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: ui/litellm-dashboard + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: ui/litellm-dashboard/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build + run: npm run build diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml new file mode 100644 index 00000000000..0c040b3ebe7 --- /dev/null +++ b/.github/workflows/test-litellm.yml @@ -0,0 +1,53 @@ +name: LiteLLM Mock Tests (folder - tests/test_litellm) + +# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs +# the same tests in parallel across 10 jobs for faster CI times. +# Kept for manual debugging only. +on: + workflow_dispatch: # Manual trigger only + # pull_request: + # branches: [ main ] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 25 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install 'pytest-xdist==3.8.0' + poetry run pip install "google-genai==1.22.0" + poetry run pip install "google-cloud-aiplatform==1.115.0" + poetry run pip install "fastapi-offline==1.7.3" + poetry run pip install "python-multipart==0.0.22" + poetry run pip install "openapi-core==0.23.0" + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + - name: Run tests + run: | + poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml new file mode 100644 index 00000000000..1b228ab76bb --- /dev/null +++ b/.github/workflows/test-mcp.yml @@ -0,0 +1,52 @@ +name: LiteLLM MCP Tests (folder - tests/mcp_tests) + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 25 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Thank You Message + run: | + echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY + echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Install dependencies + run: | + poetry lock + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install "pytest==7.3.1" + poetry run pip install "pytest-retry==1.6.3" + poetry run pip install "pytest-cov==5.0.0" + poetry run pip install "pytest-asyncio==0.21.1" + poetry run pip install "respx==0.22.0" + poetry run pip install "pydantic==2.11.0" + poetry run pip install "mcp==1.25.0" + poetry run pip install 'pytest-xdist==3.8.0' + + - name: Setup litellm-enterprise as local package + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Run MCP tests + run: | + poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5 diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml new file mode 100644 index 00000000000..429f9e1ce0a --- /dev/null +++ b/.github/workflows/test-model-map.yaml @@ -0,0 +1,20 @@ +name: Validate model_prices_and_context_window.json + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + validate-model-prices-json: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Validate model_prices_and_context_window.json + run: | + jq empty model_prices_and_context_window.json diff --git a/.github/workflows/test-proxy-e2e-azure-batches.yml b/.github/workflows/test-proxy-e2e-azure-batches.yml new file mode 100644 index 00000000000..7cbbe0b338f --- /dev/null +++ b/.github/workflows/test-proxy-e2e-azure-batches.yml @@ -0,0 +1,97 @@ +name: Proxy E2E Azure Batches Tests + +on: + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + proxy_e2e_azure_batches_tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + + services: + postgres: + image: postgres:15 + env: + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry-e2e-batches- + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy" + poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run Prisma migrations + env: + DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + run: | + cd litellm/proxy + poetry run prisma migrate deploy --schema schema.prisma + cd ../.. + + - name: Run Azure Batch E2E Tests + env: + DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm + USE_LOCAL_LITELLM: "true" + USE_MOCK_MODELS: "true" + USE_STATE_TRACKER: "true" + LITELLM_LOG: DEBUG + run: | + poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \ + -vv -s -k "test_e2e_managed_batch" \ + --tb=short \ + --maxfail=3 \ + --durations=10 diff --git a/.github/workflows/test-unit-caching-redis.yml b/.github/workflows/test-unit-caching-redis.yml new file mode 100644 index 00000000000..ca274324f2f --- /dev/null +++ b/.github/workflows/test-unit-caching-redis.yml @@ -0,0 +1,38 @@ +name: "Unit Tests: Caching (Redis)" + +# Uses cloud Redis credentials — only runs on trusted branches, not PRs. +# This prevents external PRs from accessing Redis credentials. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + caching-redis: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + # Redis-only tests that do NOT require provider API keys. + # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py, + # test_router_caching.py) are in Phase 3 integration workflows. + test-path: >- + tests/local_testing/test_dual_cache.py + tests/local_testing/test_redis_batch_optimizations.py + tests/local_testing/test_router_utils.py + workers: 2 + reruns: 2 + timeout-minutes: 20 + enable-redis: true + enable-postgres: false + secrets: + REDIS_HOST: ${{ secrets.REDIS_HOST }} + REDIS_PORT: ${{ secrets.REDIS_PORT }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml new file mode 100644 index 00000000000..2f3698fdf60 --- /dev/null +++ b/.github/workflows/test-unit-core-utils.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Core Utilities" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + core-utils: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/litellm_core_utils" + workers: 2 + reruns: 1 diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml new file mode 100644 index 00000000000..d8b30de6844 --- /dev/null +++ b/.github/workflows/test-unit-documentation.yml @@ -0,0 +1,67 @@ +name: "Unit Tests: Documentation Validation" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + documentation: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + # Run the same documentation tests that CircleCI ran (as direct Python scripts) + - name: Run documentation validation tests + run: | + poetry run python ./tests/documentation_tests/test_env_keys.py + poetry run python ./tests/documentation_tests/test_router_settings.py + poetry run python ./tests/documentation_tests/test_api_docs.py + poetry run python ./tests/documentation_tests/test_circular_imports.py diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml new file mode 100644 index 00000000000..13ae3efedba --- /dev/null +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -0,0 +1,24 @@ +name: "Unit Tests: Enterprise, Google GenAI & Routing" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + enterprise-routing: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/enterprise + tests/test_litellm/google_genai + tests/test_litellm/router_utils + tests/test_litellm/router_strategy + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml new file mode 100644 index 00000000000..2789f99d81c --- /dev/null +++ b/.github/workflows/test-unit-integrations.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Integrations (Callbacks & Logging)" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + integrations: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/integrations" + workers: 2 + reruns: 3 diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml new file mode 100644 index 00000000000..6c00272b0c8 --- /dev/null +++ b/.github/workflows/test-unit-llm-providers.yml @@ -0,0 +1,29 @@ +name: "Unit Tests: LLM Provider Transformations" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + vertex-ai: + name: Vertex AI + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms/vertex_ai" + workers: 1 + reruns: 2 + + other-providers: + name: All Other Providers + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml new file mode 100644 index 00000000000..9228decd7cc --- /dev/null +++ b/.github/workflows/test-unit-misc.yml @@ -0,0 +1,31 @@ +name: "Unit Tests: MCP, Secrets, Containers & Misc" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + misc: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/secret_managers + tests/test_litellm/a2a_protocol + tests/test_litellm/anthropic_interface + tests/test_litellm/completion_extras + tests/test_litellm/containers + tests/test_litellm/experimental_mcp_client + tests/test_litellm/images + tests/test_litellm/interactions + tests/test_litellm/passthrough + tests/test_litellm/vector_stores + tests/test_litellm/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml new file mode 100644 index 00000000000..e71821db701 --- /dev/null +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Proxy Auth & Key Management" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-auth: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml new file mode 100644 index 00000000000..bdfb6efeef1 --- /dev/null +++ b/.github/workflows/test-unit-proxy-db.yml @@ -0,0 +1,45 @@ +name: "Unit Tests: Proxy DB Operations" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + proxy-db: + strategy: + fail-fast: false + matrix: + include: + # Key generation tests must NOT run in parallel (event loop conflicts with logging worker) + - test-group: key-generation + test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" + workers: 0 + timeout: 30 + - test-group: auth-checks + test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + - test-group: remaining + test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py" + workers: 8 + timeout: 20 + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: ${{ matrix.test-path }} + workers: ${{ matrix.workers }} + reruns: 2 + timeout-minutes: ${{ matrix.timeout }} + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml new file mode 100644 index 00000000000..caff3b3ae06 --- /dev/null +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -0,0 +1,35 @@ +name: "Unit Tests: Proxy API Endpoints" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-endpoints: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/management_endpoints + tests/test_litellm/proxy/guardrails + tests/test_litellm/proxy/management_helpers + tests/test_litellm/proxy/anthropic_endpoints + tests/test_litellm/proxy/google_endpoints + tests/test_litellm/proxy/openai_files_endpoint + tests/test_litellm/proxy/response_api_endpoints + tests/test_litellm/proxy/image_endpoints + tests/test_litellm/proxy/vector_store_endpoints + tests/test_litellm/proxy/agent_endpoints + tests/test_litellm/proxy/discovery_endpoints + tests/test_litellm/proxy/health_endpoints + tests/test_litellm/proxy/public_endpoints + tests/test_litellm/proxy/prompts + tests/test_litellm/proxy/ui_crud_endpoints + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml new file mode 100644 index 00000000000..4dfbbe317ed --- /dev/null +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Proxy Infrastructure" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + proxy-infra: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: >- + tests/test_litellm/proxy/db + tests/test_litellm/proxy/middleware + tests/test_litellm/proxy/spend_tracking + tests/test_litellm/proxy/pass_through_endpoints + tests/test_litellm/proxy/_experimental + tests/test_litellm/proxy/experimental + tests/test_litellm/proxy/common_utils + tests/test_litellm/proxy/test_*.py + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml new file mode 100644 index 00000000000..a9391137263 --- /dev/null +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -0,0 +1,96 @@ +name: "Unit Tests: Proxy Legacy Tests" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + test-group: + - name: "auth-and-jwt" + path: "tests/proxy_unit_tests/test_[a-j]*.py" + - name: "key-generation" + path: "tests/proxy_unit_tests/test_[k-o]*.py" + - name: "proxy-config" + path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py" + - name: "proxy-server" + path: "tests/proxy_unit_tests/test_proxy_server.py" + - name: "proxy-server-extras" + path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py" + - name: "proxy-utils" + path: "tests/proxy_unit_tests/test_proxy_utils.py" + - name: "proxy-token-counter" + path: "tests/proxy_unit_tests/test_proxy_token_counter.py" + - name: "proxy-response-and-misc" + path: "tests/proxy_unit_tests/test_[r-t]*.py" + - name: "proxy-user-auth-and-spend" + path: "tests/proxy_unit_tests/test_[u-z]*.py" + + name: ${{ matrix.test-group.name }} + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Poetry + run: pip install 'poetry==2.3.2' + + - name: Cache Poetry dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/pypoetry + ~/.cache/pip + .venv + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} + restore-keys: | + ${{ runner.os }}-poetry- + + - name: Install dependencies + run: | + poetry config virtualenvs.in-project true + poetry install --with dev,proxy-dev --extras "proxy semantic-router" + poetry run pip install google-genai==1.22.0 \ + google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0 + + - name: Setup litellm-enterprise + run: | + poetry run pip install --force-reinstall --no-deps -e enterprise/ + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: | + poetry run pip install nodejs-wheel-binaries==24.13.1 + poetry run prisma generate --schema litellm/proxy/schema.prisma + + - name: Run tests - ${{ matrix.test-group.name }} + env: + TEST_PATH: ${{ matrix.test-group.path }} + run: | + poetry run pytest ${TEST_PATH} \ + --tb=short -vv \ + --maxfail=10 \ + -n 2 \ + --reruns 1 \ + --reruns-delay 1 \ + --dist=loadscope \ + --durations=20 diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml new file mode 100644 index 00000000000..7f3acac2803 --- /dev/null +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -0,0 +1,20 @@ +name: "Unit Tests: Responses, Caching & Types" + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + responses-caching-types: + uses: ./.github/workflows/_test-unit-base.yml + with: + test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types" + workers: 2 + reruns: 2 diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml new file mode 100644 index 00000000000..b38c82b1c24 --- /dev/null +++ b/.github/workflows/test-unit-security.yml @@ -0,0 +1,28 @@ +name: "Unit Tests: Security" + +# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. +on: + push: + branches: [main, "litellm_*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + security: + uses: ./.github/workflows/_test-unit-services-base.yml + with: + test-path: "tests/proxy_security_tests/" + workers: 1 + reruns: 2 + timeout-minutes: 20 + enable-redis: false + enable-postgres: true + secrets: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} diff --git a/.github/workflows/test_server_root_path.yml b/.github/workflows/test_server_root_path.yml new file mode 100644 index 00000000000..47636ce8e92 --- /dev/null +++ b/.github/workflows/test_server_root_path.yml @@ -0,0 +1,98 @@ +name: Test Proxy SERVER_ROOT_PATH Routing +permissions: + contents: read + +on: + pull_request: + branches: [main] + +jobs: + test-server-root-path: + runs-on: ubuntu-latest + timeout-minutes: 15 + + strategy: + matrix: + root_path: ["/api/v1", "/llmproxy"] + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12 + + - name: Build Docker image + uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14 + with: + context: . + file: ./docker/Dockerfile.non_root + tags: litellm-test:${{ github.sha }} + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Start LiteLLM container with SERVER_ROOT_PATH + run: | + docker run -d \ + --name litellm-test \ + -p 4000:4000 \ + -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + litellm-test:${{ github.sha }} \ + --detailed_debug + + - name: Wait for container to be healthy + run: | + echo "Waiting for LiteLLM to start..." + max_attempts=30 + attempt=0 + + while [ $attempt -lt $max_attempts ]; do + if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then + echo "LiteLLM started successfully" + break + fi + attempt=$((attempt + 1)) + echo "Attempt $attempt/$max_attempts - waiting for server to start..." + sleep 2 + done + + if [ $attempt -eq $max_attempts ]; then + echo "Server failed to start within timeout" + docker logs litellm-test + exit 1 + fi + + sleep 5 + + - name: Show container logs + if: always() + run: docker logs litellm-test + + - name: Test UI endpoint with root path + run: | + ROOT_PATH="${{ matrix.root_path }}" + echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/" + + for i in 1 2 3; do + content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/") + if echo "$content" | grep -q -E "(html|/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - +elif command -v shasum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c - +else + echo "No sha256 tool found; cannot verify nvm checksum"; exit 1 +fi || { echo "nvm checksum verification failed"; exit 1; } +bash "$NVM_SCRIPT" source ~/.nvm/nvm.sh nvm install v18.17.0 nvm use v18.17.0 -npm install -g npm # copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100644 index 00000000000..03ae31cd18b --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# LiteLLM Installer +# Usage: curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh +# +# NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian +# ignores the shebang when invoked as `sh` and does not support `pipefail`). +set -eu + +MIN_PYTHON_MAJOR=3 +MIN_PYTHON_MINOR=9 + +# NOTE: before merging, this must stay as "litellm[proxy]" to install from PyPI. +LITELLM_PACKAGE="litellm[proxy]" + +# ── colours ──────────────────────────────────────────────────────────────── +if [ -t 1 ]; then + BOLD='\033[1m' + GREEN='\033[38;2;78;186;101m' + GREY='\033[38;2;153;153;153m' + RESET='\033[0m' +else + BOLD='' GREEN='' GREY='' RESET='' +fi + +info() { printf "${GREY} %s${RESET}\n" "$*"; } +success() { printf "${GREEN} ✔ %s${RESET}\n" "$*"; } +header() { printf "${BOLD} %s${RESET}\n" "$*"; } +die() { printf "\n Error: %s\n\n" "$*" >&2; exit 1; } + +# ── banner ───────────────────────────────────────────────────────────────── +echo "" +cat << 'EOF' + ██╗ ██╗████████╗███████╗██╗ ██╗ ███╗ ███╗ + ██║ ██║╚══██╔══╝██╔════╝██║ ██║ ████╗ ████║ + ██║ ██║ ██║ █████╗ ██║ ██║ ██╔████╔██║ + ██║ ██║ ██║ ██╔══╝ ██║ ██║ ██║╚██╔╝██║ + ███████╗██║ ██║ ███████╗███████╗███████╗██║ ╚═╝ ██║ + ╚══════╝╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝ ╚═╝ +EOF +printf " ${BOLD}LiteLLM Installer${RESET} ${GREY}— unified gateway for 100+ LLM providers${RESET}\n\n" + +# ── OS detection ─────────────────────────────────────────────────────────── +OS="$(uname -s)" +ARCH="$(uname -m)" + +case "$OS" in + Darwin) PLATFORM="macOS ($ARCH)" ;; + Linux) PLATFORM="Linux ($ARCH)" ;; + *) die "Unsupported OS: $OS. LiteLLM supports macOS and Linux." ;; +esac + +info "Platform: $PLATFORM" + +# ── Python detection ─────────────────────────────────────────────────────── +PYTHON_BIN="" +for candidate in python3 python; do + if command -v "$candidate" >/dev/null 2>&1; then + major="$("$candidate" -c 'import sys; print(sys.version_info.major)' 2>/dev/null || true)" + minor="$("$candidate" -c 'import sys; print(sys.version_info.minor)' 2>/dev/null || true)" + if [ "${major:-0}" -ge "$MIN_PYTHON_MAJOR" ] && [ "${minor:-0}" -ge "$MIN_PYTHON_MINOR" ]; then + PYTHON_BIN="$(command -v "$candidate")" + info "Python: $("$candidate" --version 2>&1)" + break + fi + fi +done + +if [ -z "$PYTHON_BIN" ]; then + die "Python ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+ is required but not found. + Install it from https://python.org/downloads or via your package manager: + macOS: brew install python@3 + Ubuntu: sudo apt install python3 python3-pip" +fi + +# ── pip detection ────────────────────────────────────────────────────────── +if ! "$PYTHON_BIN" -m pip --version >/dev/null 2>&1; then + die "pip is not available. Install it with: + $PYTHON_BIN -m ensurepip --upgrade" +fi + +# ── install ──────────────────────────────────────────────────────────────── +echo "" +header "Installing litellm[proxy]…" +echo "" + +"$PYTHON_BIN" -m pip install --only-binary :all: --upgrade "${LITELLM_PACKAGE}" \ + || die "pip install failed. Try manually: $PYTHON_BIN -m pip install --only-binary :all: '${LITELLM_PACKAGE}'" + +# ── find the litellm binary installed by pip for this Python ─────────────── +# sysconfig.get_path('scripts') is where pip puts console scripts — reliable +# even when the Python lives in a libexec/ symlink tree (e.g. Homebrew). +SCRIPTS_DIR="$("$PYTHON_BIN" -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" +LITELLM_BIN="${SCRIPTS_DIR}/litellm" + +if [ ! -x "$LITELLM_BIN" ]; then + # Fall back to user-base bin (pip install --user) + USER_BIN="$("$PYTHON_BIN" -c 'import site; print(site.getuserbase())')/bin" + LITELLM_BIN="${USER_BIN}/litellm" +fi + +if [ ! -x "$LITELLM_BIN" ]; then + die "litellm binary not found after install. Try: $PYTHON_BIN -m pip install --user '${LITELLM_PACKAGE}'" +fi + +# ── success banner ───────────────────────────────────────────────────────── +echo "" +success "LiteLLM installed" + +installed_ver="$("$LITELLM_BIN" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" +[ -n "$installed_ver" ] && info "Version: $installed_ver" + +# ── PATH hint ────────────────────────────────────────────────────────────── +if ! command -v litellm >/dev/null 2>&1; then + info "Note: add litellm to your PATH: export PATH=\"\$PATH:${SCRIPTS_DIR}\"" +fi + +# ── launch setup wizard ──────────────────────────────────────────────────── +echo "" +printf " ${BOLD}Run the interactive setup wizard?${RESET} ${GREY}(Y/n)${RESET}: " +# /dev/tty may be unavailable in Docker/CI — default to yes if it can't be read +answer="" +if [ -r /dev/tty ]; then + read -r answer /dev/null; then - # Install nvm - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash + # Install nvm with checksum verification + NVM_VERSION="v0.40.4" + NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" + NVM_SCRIPT=$(mktemp) + trap 'rm -f "$NVM_SCRIPT"' EXIT + curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" + if command -v sha256sum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - + elif command -v shasum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c - + else + echo "No sha256 tool found; cannot verify nvm checksum"; exit 1 + fi || { echo "nvm checksum verification failed"; exit 1; } + bash "$NVM_SCRIPT" # Source nvm script in the current session export NVM_DIR="$HOME/.nvm" @@ -11,11 +23,11 @@ if ! command -v nvm &> /dev/null; then fi # Use nvm to set the required Node.js version -nvm use v18.17.0 +nvm use v20 # Check if nvm use was successful if [ $? -ne 0 ]; then - echo "Error: Failed to switch to Node.js v18.17.0. Deployment aborted." + echo "Error: Failed to switch to Node.js v20. Deployment aborted." exit 1 fi @@ -43,6 +55,8 @@ if [ $? -eq 0 ]; then # Copy the contents of the output directory to the specified destination cp -r ./out/* "$destination_dir" + rm -rf ./out + echo "Deployment completed." else echo "Build failed. Deployment aborted." diff --git a/ui/litellm-dashboard/build_ui_custom_path.sh b/ui/litellm-dashboard/build_ui_custom_path.sh index f947f87d3b7..a92927f8ea7 100755 --- a/ui/litellm-dashboard/build_ui_custom_path.sh +++ b/ui/litellm-dashboard/build_ui_custom_path.sh @@ -12,8 +12,20 @@ UI_BASE_PATH="$1" # Check if nvm is not installed if ! command -v nvm &> /dev/null; then - # Install nvm - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash + # Install nvm with checksum verification + NVM_VERSION="v0.40.4" + NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" + NVM_SCRIPT=$(mktemp) + trap 'rm -f "$NVM_SCRIPT"' EXIT + curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" + if command -v sha256sum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - + elif command -v shasum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c - + else + echo "No sha256 tool found; cannot verify nvm checksum"; exit 1 + fi || { echo "nvm checksum verification failed"; exit 1; } + bash "$NVM_SCRIPT" # Source nvm script in the current session export NVM_DIR="$HOME/.nvm"